From 806d1a892e440054bca6765ceb70a10064320711 Mon Sep 17 00:00:00 2001 From: Vidas Date: Wed, 10 Sep 2025 18:05:18 +0300 Subject: [PATCH 01/12] ggml-et: Add RMS_NORM op --- ggml/src/ggml-et/ggml-et-cpu-compare.cpp | 8 + tests/test-backend-ops.cpp | 10327 --------------------- 2 files changed, 8 insertions(+), 10327 deletions(-) delete mode 100644 tests/test-backend-ops.cpp diff --git a/ggml/src/ggml-et/ggml-et-cpu-compare.cpp b/ggml/src/ggml-et/ggml-et-cpu-compare.cpp index b37f6d261d97..15c710cc2d5f 100644 --- a/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +++ b/ggml/src/ggml-et/ggml-et-cpu-compare.cpp @@ -311,6 +311,14 @@ bool ggml_et_cpu_compare_compute_and_check(ggml_et_cpu_compare_ctx * ct ctx->cpu_dst = ggml_set_rows(ctx->ggml_ctx, cpu_dst_base, ctx->cpu_src0, ctx->cpu_src1); } break; + case GGML_OP_RMS_NORM: + // Extract epsilon parameter from op_params (stored as float) + { + float eps; + memcpy(&eps, node->op_params, sizeof(float)); + ctx->cpu_dst = ggml_rms_norm(ctx->ggml_ctx, ctx->cpu_src0, eps); + } + break; default: GGML_LOG_ERROR("ET: Unsupported operation %s for CPU comparison\n", ggml_op_name(op)); return false; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp deleted file mode 100644 index 084344fb25d7..000000000000 --- a/tests/test-backend-ops.cpp +++ /dev/null @@ -1,10327 +0,0 @@ -// This file defines tests for various GGML ops and backends. -// For the forward pass it asserts that the results of multiple backends computing the same GGML ops are consistent. -// For the backward pass it asserts that the gradients from backpropagation are consistent -// with the gradients obtained via the method of finite differences ("grad" mode, this is optional). -// It is also possible to check the performance ("perf" mode). -// -// this file has three sections: Section 1 does general setup, section 2 defines the GGML ops to be tested, -// and section 3 defines which tests to run. -// Quick start for adding a new GGML op: Go to section 2 and create a struct that inherits from test_case, -// then go to section 3 and add an instantiation of your struct. - - -// ############################## -// ## Section 1: General Setup ## -// ############################## - - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __EMSCRIPTEN__ -# define N_THREADS 1 -#else -# define N_THREADS std::thread::hardware_concurrency() -#endif - -static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { - size_t nels = ggml_nelements(tensor); - std::vector data(nels); - { - // parallel initialization - static const size_t n_threads = N_THREADS; - - auto init_thread = [&](size_t start, size_t end) { - thread_local std::default_random_engine gen(std::random_device{}()); - std::uniform_real_distribution distribution(min, max); - for (size_t i = start; i < end; i++) { - data[i] = distribution(gen); - } - }; - - if (n_threads == 1) { - init_thread(0, nels); - } else { - std::vector> tasks; - tasks.reserve(n_threads); - for (size_t i = 0; i < n_threads; i++) { - size_t start = i*nels/n_threads; - size_t end = (i+1)*nels/n_threads; - tasks.push_back(std::async(std::launch::async, init_thread, start, end)); - } - for (auto & t : tasks) { - t.get(); - } - } - } - - if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_I32) { - ggml_backend_tensor_set(tensor, data.data(), 0, nels * sizeof(float)); - } else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) { - GGML_ASSERT(nels % ggml_blck_size(tensor->type) == 0); - - // dummy importance matrix - std::vector imatrix(tensor->ne[0], 1.0f); - const float * im = imatrix.data(); - if (!ggml_quantize_requires_imatrix(tensor->type)) { - // when the imatrix is optional, we want to test both quantization with and without imatrix - // use one of the random numbers to decide - if (data[0] > 0.5f*(min + max)) { - im = nullptr; - } - } - - std::vector dataq(ggml_row_size(tensor->type, nels)); - { - // parallel quantization by block - size_t blck_size = ggml_blck_size(tensor->type); - size_t n_blocks = nels / blck_size; - - auto quantize_thread = [&](size_t start, size_t end) { - ggml_quantize_chunk(tensor->type, data.data(), dataq.data(), - start * blck_size, end - start, blck_size, im); - }; - - const size_t min_blocks_per_thread = 1; - const size_t n_quant_threads = std::min(std::max(N_THREADS/2, 1), - std::max(1, n_blocks / min_blocks_per_thread)); - - if (n_quant_threads == 1) { - // single-threaded quantization: do all blocks in the current thread - quantize_thread(0, n_blocks); - } else { - std::vector> tasks; - tasks.reserve(n_quant_threads); - for (size_t i = 0; i < n_quant_threads; i++) { - size_t start = i*n_blocks/n_quant_threads; - size_t end = (i+1)*n_blocks/n_quant_threads; - tasks.push_back(std::async(std::launch::async, quantize_thread, start, end)); - } - for (auto & t : tasks) { - t.get(); - } - } - } - ggml_backend_tensor_set(tensor, dataq.data(), 0, dataq.size()); - } else if (tensor->type == GGML_TYPE_I8 || tensor->type == GGML_TYPE_I16) { - // This is going to create some weird integers though. - ggml_backend_tensor_set(tensor, data.data(), 0, nels * ggml_type_size(tensor->type)); - } else if (tensor->type == GGML_TYPE_I64) { - // Integers with a size of 8 bytes can be set by mirroring the float data, the specific values are again not really meaningful. - const size_t nbytes_half = nels * sizeof(float); - ggml_backend_tensor_set(tensor, data.data(), 0*nbytes_half, nbytes_half); - ggml_backend_tensor_set(tensor, data.data(), 1*nbytes_half, nbytes_half); - } else { - GGML_ABORT("fatal error"); - } -} - -// generate an F16 mask where certain blocks are randomly masked with -INF value -static void init_tensor_kq_mask(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { - GGML_ASSERT(tensor->type == GGML_TYPE_F16); - - GGML_TENSOR_LOCALS( int32_t, ne, tensor, ne); - - std::vector data_f32(ne0*ne1*ne2*ne3); - std::vector data_f16(ne0*ne1*ne2*ne3); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_real_distribution dis(min, max); - - for (size_t i = 0; i < data_f32.size(); i++) { - data_f32[i] = dis(gen); - } - - // block size - const int blck0 = 128; - const int blck1 = 64; - - // number of INF/zero blocks - const int n_inf_zero_blocks = 0.2*(ne0*ne1*ne2*ne3)/(blck0*blck1); - - for (int b = 0; b < n_inf_zero_blocks; b++) { - const int p3 = (rd() % ne3); - const int p2 = (rd() % ne2); - const int p1 = (rd() % ne1); - const int p0 = (rd() % ne0); - - bool inf = rd() & 1; - - for (int i1 = 0; i1 < blck1 && p1 + i1 < ne1; i1++) { - const int idx = p3*ne2*ne1*ne0 + p2*ne1*ne0 + (p1 + i1)*ne0 + p0; - - for (int i0 = 0; i0 < blck0 && p0 + i0 < ne0; i0++) { - data_f32[idx + i0] = inf ? -INFINITY : 0.0f; - } - } - } - - ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), ne0*ne1*ne2*ne3); - - ggml_backend_tensor_set(tensor, data_f16.data(), 0, data_f16.size()*sizeof(ggml_fp16_t)); -} - -// generate a lower triangular matrix -static void init_tensor_tril(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { - GGML_ASSERT(tensor->type == GGML_TYPE_F32); - GGML_ASSERT(tensor->ne[0] == tensor->ne[1]); - - GGML_TENSOR_LOCALS(int32_t, ne, tensor, ne); - GGML_TENSOR_LOCALS(size_t, nb, tensor, nb); - - std::vector data_f32(ne0*ne1*ne2*ne3); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_real_distribution dis(min, max); - - for (int64_t i3 = 0; i3 < ne3; i3++) { - for (int64_t i2 = 0; i2 < ne2; i2++) { - for (int64_t i1 = 0; i1 < ne1; i1++) { - for (int64_t i0 = 0; i0 < ne0; i0++) { - int64_t idx = (i0 * nb0 + i1 * nb1 + i2 * nb2 + i3 * nb3) / sizeof(float); - if (i0 <= i1) { - data_f32[idx] = dis(gen); - } else { - data_f32[idx] = 0.0f; - } - } - } - } - } - - ggml_backend_tensor_set(tensor, data_f32.data(), 0, ggml_nbytes(tensor)); -} - -static std::vector tensor_to_float(const ggml_tensor * t) { - std::vector tv; - tv.reserve(ggml_nelements(t)); - - std::vector buf(ggml_nbytes(t)); - ggml_backend_tensor_get(t, buf.data(), 0, ggml_nbytes(t)); - - const auto * tt = ggml_get_type_traits(t->type); - size_t bs = ggml_blck_size(t->type); - std::vector vq(ggml_blck_size(t->type)); - bool quantized = ggml_is_quantized(t->type); - - // access elements by index to avoid gaps in views - for (int64_t i3 = 0; i3 < t->ne[3]; i3++) { - for (int64_t i2 = 0; i2 < t->ne[2]; i2++) { - for (int64_t i1 = 0; i1 < t->ne[1]; i1++) { - for (int64_t i0 = 0; i0 < t->ne[0]; i0 += bs) { - size_t i = i3*t->nb[3] + i2*t->nb[2] + i1*t->nb[1] + i0/bs*t->nb[0]; - if (t->type == GGML_TYPE_F16) { - tv.push_back(ggml_fp16_to_fp32(*(ggml_fp16_t*)&buf[i])); - } else if (t->type == GGML_TYPE_BF16) { - tv.push_back(ggml_bf16_to_fp32(*(ggml_bf16_t*)&buf[i])); - } else if (t->type == GGML_TYPE_F32) { - tv.push_back(*(float *) &buf[i]); - } else if (t->type == GGML_TYPE_I64) { - tv.push_back((float)*(int64_t *) &buf[i]); - } else if (t->type == GGML_TYPE_I32) { - tv.push_back((float)*(int32_t *) &buf[i]); - } else if (t->type == GGML_TYPE_I16) { - tv.push_back((float)*(int16_t *) &buf[i]); - } else if (t->type == GGML_TYPE_I8) { - tv.push_back((float)*(int8_t *) &buf[i]); - } else if (quantized) { - tt->to_float(&buf[i], vq.data(), bs); - tv.insert(tv.end(), vq.begin(), vq.end()); - } else { - GGML_ABORT("fatal error"); - } - } - } - } - } - - return tv; -} - -// normalized mean squared error = mse(a, b) / mse(a, 0) -static double nmse(const float * a, const float * b, size_t n) { - double mse_a_b = 0.0; - double mse_a_0 = 0.0; - - for (size_t i = 0; i < n; i++) { - float a_i = a[i]; - float b_i = b[i]; - - mse_a_b += (a_i - b_i) * (a_i - b_i); - mse_a_0 += a_i * a_i; - } - - return mse_a_b / mse_a_0; -} - -// difference between 2 sets (Jaccard distance, 0 - no difference, 1 - no overlap) -template -static double jdst(const T * a, const T * b, size_t n) { - std::unordered_map set_a; - std::unordered_map set_b; - - for (size_t i = 0; i < n; ++i) { - set_a[a[i]]++; - set_b[b[i]]++; - } - - size_t diff = 0; - - for (const auto & p : set_a) { - const int64_t na = p.second; - const int64_t nb = set_b.find(p.first) != set_b.end() ? set_b.at(p.first) : 0; - - diff += std::abs(na - nb); - } - - for (const auto & p : set_b) { - if (set_a.find(p.first) == set_a.end()) { - diff += p.second; - } - } - - return (double) diff / (2*n); -} - -// maximum absolute asymmetry between a and b -// asymmetry: (a - b) / (a + b) -// This is more stable than relative error if one of the values fluctuates towards zero. -// n: number of values to compare. -// expected_vals: optional vector of expected values for a. If expected_vals is not empty, filter out all comparisons where -// a does not match any of the expected values. Needed for noncontinuous gradients where the numerical calculation can fail. -static double mean_abs_asymm(const float * a, const float * b, const size_t n, const std::vector & expected_vals) { - double sum = 0.0f; - - size_t nvalid = 0; - for (size_t i = 0; i < n; i++) { - if (!expected_vals.empty()) { - bool matches_any = false; - for (const float & ev : expected_vals) { - if (fabsf(a[i] - ev) < 1e-3f) { - matches_any = true; - break; - } - } - if (!matches_any) { - continue; - } - } - - const float asymm = (a[i] - b[i]) / (a[i] + b[i]); - - sum += fabsf(asymm); - nvalid++; - } - - return sum/nvalid; -} - -// utils for printing the variables of the test cases - -static std::string var_to_str(const std::string & x) { - return x; -} - -template -static std::string var_to_str(const T & x) { - return std::to_string(x); -} - -template -static std::string var_to_str(const T (&x)[N]) { - std::string s = "["; - for (size_t i = 0; i < N; i++) { - if (i > 0) { - s += ","; - } - s += var_to_str(x[i]); - } - s += "]"; - return s; -} - -template -static std::string var_to_str(const std::array & x) { - std::string s = "["; - for (size_t i = 0; i < N; i++) { - if (i > 0) { - s += ","; - } - s += var_to_str(x[i]); - } - s += "]"; - return s; -} - -static std::string var_to_str(ggml_type type) { - return ggml_type_name(type); -} - -static std::string var_to_str(ggml_prec prec) { - return prec == GGML_PREC_F32 ? "f32" : "def"; -} - -static std::string var_to_str(ggml_op_pool pool) { - switch (pool) { - case GGML_OP_POOL_AVG: return "avg"; - case GGML_OP_POOL_MAX: return "max"; - default: return std::to_string(pool); - } -} - -static std::string var_to_str(ggml_scale_mode mode) { - std::string str; - switch (mode & 0xFF) { - case GGML_SCALE_MODE_NEAREST: str = "nearest"; break; - case GGML_SCALE_MODE_BILINEAR: str = "bilinear"; break; - case GGML_SCALE_MODE_BICUBIC: str = "bicubic"; break; - default: str = std::to_string(mode); break; - } - if (mode & GGML_SCALE_FLAG_ALIGN_CORNERS) { - str += "|align_corners"; - } - if (mode & GGML_SCALE_FLAG_ANTIALIAS) { - str += "|antialias"; - } - return str; -} - -#define VAR_TO_STR(x) (#x "=" + var_to_str(x)) - -#define VARS_TO_STR1(a) VAR_TO_STR(a) -#define VARS_TO_STR2(a, b) VAR_TO_STR(a) + "," + VAR_TO_STR(b) -#define VARS_TO_STR3(a, b, c) VAR_TO_STR(a) + "," + VARS_TO_STR2(b, c) -#define VARS_TO_STR4(a, b, c, d) VAR_TO_STR(a) + "," + VARS_TO_STR3(b, c, d) -#define VARS_TO_STR5(a, b, c, d, e) VAR_TO_STR(a) + "," + VARS_TO_STR4(b, c, d, e) -#define VARS_TO_STR6(a, b, c, d, e, f) VAR_TO_STR(a) + "," + VARS_TO_STR5(b, c, d, e, f) -#define VARS_TO_STR7(a, b, c, d, e, f, g) VAR_TO_STR(a) + "," + VARS_TO_STR6(b, c, d, e, f, g) -#define VARS_TO_STR8(a, b, c, d, e, f, g, h) VAR_TO_STR(a) + "," + VARS_TO_STR7(b, c, d, e, f, g, h) -#define VARS_TO_STR9(a, b, c, d, e, f, g, h, i) VAR_TO_STR(a) + "," + VARS_TO_STR8(b, c, d, e, f, g, h, i) -#define VARS_TO_STR10(a, b, c, d, e, f, g, h, i, j) VAR_TO_STR(a) + "," + VARS_TO_STR9(b, c, d, e, f, g, h, i, j) -#define VARS_TO_STR11(a, b, c, d, e, f, g, h, i, j, k) VAR_TO_STR(a) + "," + VARS_TO_STR10(b, c, d, e, f, g, h, i, j, k) -#define VARS_TO_STR12(a, b, c, d, e, f, g, h, i, j, k, l) VAR_TO_STR(a) + "," + VARS_TO_STR11(b, c, d, e, f, g, h, i, j, k, l) -#define VARS_TO_STR13(a, b, c, d, e, f, g, h, i, j, k, l, m) VAR_TO_STR(a) + "," + VARS_TO_STR12(b, c, d, e, f, g, h, i, j, k, l, m) -#define VARS_TO_STR14(a, b, c, d, e, f, g, h, i, j, k, l, m, n) VAR_TO_STR(a) + "," + VARS_TO_STR13(b, c, d, e, f, g, h, i, j, k, l, m, n) -#define VARS_TO_STR15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) VAR_TO_STR(a) + "," + VARS_TO_STR14(b, c, d, e, f, g, h, i, j, k, l, m, n, o) -#define VARS_TO_STR16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) VAR_TO_STR(a) + "," + VARS_TO_STR15(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) - -#ifdef GGML_USE_SYCL -static bool inline _isinf(float f) { - return (*(uint32_t *)&f & 0x7fffffff) == 0x7f800000; -} -#else -static bool inline _isinf(float f) { return std::isinf(f); } -#endif - -// accept FLT_MAX as infinity -static bool isinf_or_max(float f) { - return _isinf(f) || f == FLT_MAX || f == -FLT_MAX; -} - -static bool ggml_is_view_op(enum ggml_op op) { - return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE; -} - -static bool backend_has_feature(ggml_backend_t backend, const char * feature_name) { - ggml_backend_dev_t dev = ggml_backend_get_device(backend); - ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); - - auto get_features = (ggml_backend_get_features_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_get_features"); - if (!get_features) { - return false; - } - - const ggml_backend_feature * features = get_features(reg); - if (!features) { - return false; - } - - for (const ggml_backend_feature * f = features; f->name; ++f) { - if (strcmp(f->name, feature_name) == 0 && strcmp(f->value, "1") == 0) { - return true; - } - } - return false; -} - -enum test_mode { - MODE_TEST, - MODE_PERF, - MODE_GRAD, - MODE_SUPPORT, -}; - -// Output format support similar to llama-bench -enum output_formats { CONSOLE, SQL, CSV }; - -static const char * output_format_str(output_formats format) { - switch (format) { - case CONSOLE: - return "console"; - case SQL: - return "sql"; - case CSV: - return "csv"; - default: - GGML_ABORT("invalid output format"); - } -} - -static bool output_format_from_str(const std::string & s, output_formats & format) { - if (s == "console") { - format = CONSOLE; - } else if (s == "sql") { - format = SQL; - } else if (s == "csv") { - format = CSV; - } else { - return false; - } - return true; -} - -static std::string test_time_now() { - time_t t = time(NULL); - struct tm tm_buf; -#ifdef _WIN32 - if (gmtime_s(&tm_buf, &t) != 0) { - return ""; - } -#else - if (gmtime_r(&t, &tm_buf) == nullptr) { - return ""; - } -#endif - char buf[32]; - if (std::strftime(buf, sizeof(buf), "%FT%TZ", &tm_buf) == 0) { - return ""; - } - return buf; -} - -// Test result structure for SQL output -struct test_result { - std::string test_time; - std::string build_commit; - std::string backend_name; - std::string op_name; - std::string op_params; - std::string test_mode; - bool supported; - bool passed; - std::string error_message; - double time_us; - double flops; - double bandwidth_gb_s; - size_t memory_kb; - int n_runs; - std::string device_description; - std::string backend_reg_name; - - test_result() { - // Initialize with default values - time_us = 0.0; - flops = 0.0; - bandwidth_gb_s = 0.0; - memory_kb = 0; - n_runs = 0; - supported = false; - passed = false; - - test_time = test_time_now(); - - // Set build info - build_commit = ggml_commit(); - } - - test_result(const std::string & backend_name, const std::string & op_name, const std::string & op_params, - const std::string & test_mode, bool supported, bool passed, const std::string & error_message = "", - double time_us = 0.0, double flops = 0.0, double bandwidth_gb_s = 0.0, size_t memory_kb = 0, - int n_runs = 0, const std::string & device_description = "", const std::string & backend_reg_name = "") : - backend_name(backend_name), - op_name(op_name), - op_params(op_params), - test_mode(test_mode), - supported(supported), - passed(passed), - error_message(error_message), - time_us(time_us), - flops(flops), - bandwidth_gb_s(bandwidth_gb_s), - memory_kb(memory_kb), - n_runs(n_runs), - device_description(device_description), - backend_reg_name(backend_reg_name) { - test_time = test_time_now(); - - // Set build info - build_commit = ggml_commit(); - } - - static const std::vector & get_fields() { - static const std::vector fields = { - "test_time", "build_commit", "backend_name", "op_name", "op_params", "test_mode", "supported", - "passed", "error_message", "time_us", "flops", "bandwidth_gb_s", "memory_kb", "n_runs", - "device_description", "backend_reg_name" - }; - return fields; - } - - enum field_type { STRING, BOOL, INT, FLOAT }; - - static field_type get_field_type(const std::string & field) { - if (field == "supported" || field == "passed") { - return BOOL; - } - if (field == "memory_kb" || field == "n_runs") { - return INT; - } - if (field == "time_us" || field == "flops" || field == "bandwidth_gb_s") { - return FLOAT; - } - return STRING; - } - - std::vector get_values() const { - return { test_time, - build_commit, - backend_name, - op_name, - op_params, - test_mode, - std::to_string(supported), - std::to_string(passed), - error_message, - std::to_string(time_us), - std::to_string(flops), - std::to_string(bandwidth_gb_s), - std::to_string(memory_kb), - std::to_string(n_runs), - device_description, - backend_reg_name }; - } -}; - -// Printer classes for different output formats -enum class test_status_t { NOT_SUPPORTED, OK, FAIL, SKIPPED }; - -struct test_operation_info { - std::string op_name; - std::string op_params; - std::string backend_name; - test_status_t status = test_status_t::OK; - std::string failure_reason; - - // Additional information fields that were previously in separate structs - std::string error_component; - std::string error_details; - - // Gradient info - int64_t gradient_index = -1; - std::string gradient_param_name; - float gradient_value = 0.0f; - - // MAA error info - double maa_error = 0.0; - double maa_threshold = 0.0; - - // Flags for different types of information - bool has_error = false; - bool has_gradient_info = false; - bool has_maa_error = false; - bool is_compare_failure = false; - bool is_large_tensor_skip = false; - - test_operation_info() = default; - - test_operation_info(const std::string & op_name, const std::string & op_params, const std::string & backend_name, - test_status_t status = test_status_t::OK, const std::string & failure_reason = "") : - op_name(op_name), - op_params(op_params), - backend_name(backend_name), - status(status), - failure_reason(failure_reason) {} - - // Set error information - void set_error(const std::string & component, const std::string & details) { - has_error = true; - error_component = component; - error_details = details; - if (status == test_status_t::OK) { - status = test_status_t::FAIL; - } - } - - // Set gradient information - void set_gradient_info(int64_t index, const std::string & param_name, float value) { - has_gradient_info = true; - gradient_index = index; - gradient_param_name = param_name; - gradient_value = value; - if (status == test_status_t::OK) { - status = test_status_t::FAIL; - } - } - - // Set MAA error information - void set_maa_error(double error, double threshold) { - has_maa_error = true; - maa_error = error; - maa_threshold = threshold; - if (status == test_status_t::OK) { - status = test_status_t::FAIL; - } - } - - // Set compare failure - void set_compare_failure() { - is_compare_failure = true; - if (status == test_status_t::OK) { - status = test_status_t::FAIL; - } - } - - // Set large tensor skip - void set_large_tensor_skip() { is_large_tensor_skip = true; } -}; - -struct test_summary_info { - size_t tests_passed; - size_t tests_total; - bool is_backend_summary = false; // true for backend summary, false for test summary - - test_summary_info() = default; - - test_summary_info(size_t tests_passed, size_t tests_total, bool is_backend_summary = false) : - tests_passed(tests_passed), - tests_total(tests_total), - is_backend_summary(is_backend_summary) {} -}; - -struct testing_start_info { - size_t device_count; - - testing_start_info() = default; - - testing_start_info(size_t device_count) : device_count(device_count) {} -}; - -struct backend_init_info { - size_t device_index; - size_t total_devices; - std::string device_name; - bool skipped = false; - std::string skip_reason; - std::string description; - size_t memory_total_mb = 0; - size_t memory_free_mb = 0; - bool has_memory_info = false; - - backend_init_info() = default; - - backend_init_info(size_t device_index, size_t total_devices, const std::string & device_name, bool skipped = false, - const std::string & skip_reason = "", const std::string & description = "", - size_t memory_total_mb = 0, size_t memory_free_mb = 0, bool has_memory_info = false) : - device_index(device_index), - total_devices(total_devices), - device_name(device_name), - skipped(skipped), - skip_reason(skip_reason), - description(description), - memory_total_mb(memory_total_mb), - memory_free_mb(memory_free_mb), - has_memory_info(has_memory_info) {} -}; - -struct backend_status_info { - std::string backend_name; - test_status_t status; - - backend_status_info() = default; - - backend_status_info(const std::string & backend_name, test_status_t status) : - backend_name(backend_name), - status(status) {} -}; - -struct overall_summary_info { - size_t backends_passed; - size_t backends_total; - bool all_passed; - - overall_summary_info() = default; - - overall_summary_info(size_t backends_passed, size_t backends_total, bool all_passed) : - backends_passed(backends_passed), - backends_total(backends_total), - all_passed(all_passed) {} -}; - -struct printer { - virtual ~printer() {} - - FILE * fout = stdout; - - virtual void print_header() {} - - virtual void print_test_result(const test_result & result) = 0; - - virtual void print_footer() {} - - virtual void print_operation(const test_operation_info & info) { (void) info; } - - virtual void print_summary(const test_summary_info & info) { (void) info; } - - virtual void print_testing_start(const testing_start_info & info) { (void) info; } - - virtual void print_backend_init(const backend_init_info & info) { (void) info; } - - virtual void print_backend_status(const backend_status_info & info) { (void) info; } - - virtual void print_overall_summary(const overall_summary_info & info) { (void) info; } - - virtual void print_failed_tests(const std::vector & failed_tests) { (void) failed_tests; } -}; - -struct console_printer : public printer { - void print_test_result(const test_result & result) override { - if (result.test_mode == "test") { - print_test_console(result); - } else if (result.test_mode == "perf") { - print_perf_console(result); - } else if (result.test_mode == "support") { - print_support_console(result); - } - } - - void print_operation(const test_operation_info & info) override { - printf(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str()); - fflush(stdout); - - // Handle large tensor skip first - if (info.is_large_tensor_skip) { - printf("skipping large tensors for speed \n"); - return; - } - - // Handle not supported status - if (info.status == test_status_t::NOT_SUPPORTED) { - if (!info.failure_reason.empty()) { - printf("not supported [%s]\n", info.failure_reason.c_str()); - } else { - printf("not supported [%s]\n", info.backend_name.c_str()); - } - return; - } - - // Handle errors and additional information - if (info.has_error) { - if (info.error_component == "allocation") { - fprintf(stderr, "failed to allocate tensors [%s] ", info.backend_name.c_str()); - } else if (info.error_component == "backend") { - fprintf(stderr, " Failed to initialize %s backend\n", info.backend_name.c_str()); - } else { - fprintf(stderr, "Error in %s: %s\n", info.error_component.c_str(), info.error_details.c_str()); - } - } - - // Handle gradient info - if (info.has_gradient_info) { - printf("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", info.op_name.c_str(), info.gradient_index, - info.gradient_param_name.c_str(), info.gradient_value); - } - - // Handle MAA error - if (info.has_maa_error) { - printf("[%s] MAA = %.9f > %.9f ", info.op_name.c_str(), info.maa_error, info.maa_threshold); - } - - // Handle compare failure - if (info.is_compare_failure) { - printf("compare failed "); - } - - // Print final status - if (info.status == test_status_t::OK) { - printf("\033[1;32mOK\033[0m\n"); - } else { - printf("\033[1;31mFAIL\033[0m\n"); - } - } - - void print_summary(const test_summary_info & info) override { - if (info.is_backend_summary) { - printf("%zu/%zu backends passed\n", info.tests_passed, info.tests_total); - } else { - printf(" %zu/%zu tests passed\n", info.tests_passed, info.tests_total); - } - } - - void print_backend_status(const backend_status_info & info) override { - printf(" Backend %s: ", info.backend_name.c_str()); - if (info.status == test_status_t::OK) { - printf("\033[1;32mOK\033[0m\n"); - } else { - printf("\033[1;31mFAIL\033[0m\n"); - } - } - - void print_testing_start(const testing_start_info & info) override { - printf("Testing %zu devices\n\n", info.device_count); - } - - void print_backend_init(const backend_init_info & info) override { - printf("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str()); - - if (info.skipped) { - printf(" %s\n", info.skip_reason.c_str()); - return; - } - - if (!info.description.empty()) { - printf(" Device description: %s\n", info.description.c_str()); - } - - if (info.has_memory_info) { - printf(" Device memory: %zu MB (%zu MB free)\n", info.memory_total_mb, info.memory_free_mb); - } - - printf("\n"); - } - - void print_overall_summary(const overall_summary_info & info) override { - printf("%zu/%zu backends passed\n", info.backends_passed, info.backends_total); - if (info.all_passed) { - printf("\033[1;32mOK\033[0m\n"); - } else { - printf("\033[1;31mFAIL\033[0m\n"); - } - } - - void print_failed_tests(const std::vector & failed_tests) override { - if (failed_tests.empty()) { - return; - } - - printf("\nFailing tests:\n"); - for (const auto & test_name : failed_tests) { - printf(" %s\n", test_name.c_str()); - } - } - - private: - void print_test_console(const test_result & result) { - printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); - fflush(stdout); - - if (!result.supported) { - printf("not supported [%s] ", result.backend_name.c_str()); - printf("\n"); - return; - } - - if (result.passed) { - printf("\033[1;32mOK\033[0m\n"); - } else { - printf("\033[1;31mFAIL\033[0m\n"); - } - } - - void print_perf_console(const test_result & result) { - int len = printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); - fflush(stdout); - - if (!result.supported) { - printf("not supported\n"); - return; - } - - // align while also leaving some margin for variations in parameters - int align = 8; - int last = (len + align - 1) / align * align; - if (last - len < 5) { - last += align; - } - printf("%*s", last - len, ""); - - printf(" %8d runs - %8.2f us/run - ", result.n_runs, result.time_us); - - if (result.flops > 0) { - auto format_flops = [](double flops) -> std::string { - char buf[256]; - if (flops >= 1e12) { - snprintf(buf, sizeof(buf), "%6.2f TFLOP", flops / 1e12); - } else if (flops >= 1e9) { - snprintf(buf, sizeof(buf), "%6.2f GFLOP", flops / 1e9); - } else if (flops >= 1e6) { - snprintf(buf, sizeof(buf), "%6.2f MFLOP", flops / 1e6); - } else { - snprintf(buf, sizeof(buf), "%6.2f kFLOP", flops / 1e3); - } - return buf; - }; - uint64_t op_flops_per_run = result.flops * result.time_us / 1e6; - printf("%s/run - \033[1;34m%sS\033[0m", format_flops(op_flops_per_run).c_str(), - format_flops(result.flops).c_str()); - } else { - printf("%8zu kB/run - \033[1;34m%7.2f GB/s\033[0m", result.memory_kb, result.bandwidth_gb_s); - } - printf("\n"); - } - - void print_support_console(const test_result & result) { - printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); - fflush(stdout); - - if (result.supported) { - printf("\033[1;32mSUPPORTED\033[0m\n"); - } else { - printf("\033[1;31mNOT SUPPORTED\033[0m\n"); - } - } -}; - -struct sql_printer : public printer { - static std::string get_sql_field_type(const std::string & field) { - switch (test_result::get_field_type(field)) { - case test_result::STRING: - return "TEXT"; - case test_result::BOOL: - case test_result::INT: - return "INTEGER"; - case test_result::FLOAT: - return "REAL"; - default: - GGML_ABORT("invalid field type"); - } - } - - void print_header() override { - std::vector fields = test_result::get_fields(); - fprintf(fout, "CREATE TABLE IF NOT EXISTS test_backend_ops (\n"); - for (size_t i = 0; i < fields.size(); i++) { - fprintf(fout, " %s %s%s\n", fields[i].c_str(), get_sql_field_type(fields[i]).c_str(), - i < fields.size() - 1 ? "," : ""); - } - fprintf(fout, ");\n\n"); - } - - void print_test_result(const test_result & result) override { - fprintf(fout, "INSERT INTO test_backend_ops ("); - std::vector fields = test_result::get_fields(); - for (size_t i = 0; i < fields.size(); i++) { - fprintf(fout, "%s%s", fields[i].c_str(), i < fields.size() - 1 ? ", " : ""); - } - fprintf(fout, ") VALUES ("); - std::vector values = result.get_values(); - for (size_t i = 0; i < values.size(); i++) { - fprintf(fout, "'%s'%s", values[i].c_str(), i < values.size() - 1 ? ", " : ""); - } - fprintf(fout, ");\n"); - } -}; - -struct csv_printer : public printer { - void print_header() override { - - std::vector fields = test_result::get_fields(); - std::vector fields_csv = get_fields_csv(); - for (size_t i = 0; i < fields.size(); i++) { - if (std::find(std::begin(fields_csv), std::end(fields_csv), fields[i]) == std::end(fields_csv)) { - continue; - } - printf("\"%s\"%s", fields[i].c_str(), i < fields.size() - 1 ? "," : ""); - } - printf("\n"); - } - - void print_test_result(const test_result & result) override { - - std::vector values = result.get_values(); - std::vector fields = test_result::get_fields(); - std::vector fields_csv = get_fields_csv(); - - for (size_t i = 0; i < values.size(); i++) { - - if (std::find(std::begin(fields_csv), std::end(fields_csv), fields[i]) == std::end(fields_csv)) { - continue; - } - - // Escape quotes and wrap in quotes for CSV - std::string escaped_value = values[i]; - size_t pos = 0; - while ((pos = escaped_value.find("\"", pos)) != std::string::npos) { - escaped_value.replace(pos, 1, "\"\""); - pos += 2; - } - printf("\"%s\"%s", escaped_value.c_str(), i < values.size() - 1 ? "," : ""); - } - printf("\n"); - } - - static std::vector get_fields_csv() { - return { - "op_name", - "op_params", - "supported", - "error_message", - "test_mode", - "backend_reg_name", - "backend_name", - }; - } - -}; - -static std::unique_ptr create_printer(output_formats format) { - switch (format) { - case CONSOLE: - return std::make_unique(); - case SQL: - return std::make_unique(); - case CSV: - return std::make_unique(); - } - GGML_ABORT("invalid output format"); -} - -static std::mutex g_test_output_mutex; - -static void print_test_result_locked(printer * output_printer, const test_result & result) { - if (output_printer == nullptr) { - return; - } - - std::lock_guard guard(g_test_output_mutex); - output_printer->print_test_result(result); -} - -struct test_case { - virtual ~test_case() {} - - virtual std::string op_desc(ggml_tensor * t) { - return ggml_op_desc(t); - } - - virtual std::string vars() { - return ""; - } - - virtual ggml_tensor * build_graph(ggml_context * ctx) = 0; - virtual ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) { - GGML_UNUSED(ctx_weights); - return build_graph(ctx); - } - - virtual double max_nmse_err() { - return 1e-7; - } - - virtual double max_nmse_err(ggml_backend_t backend) { - ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); - // See https://github.com/ggml-org/llama.cpp/pull/22976 for explanation. - if (contains_f16 && strcmp(ggml_backend_reg_name(reg), "WebGPU") == 0) { - return std::max(max_nmse_err(), 1e-6); - } - return max_nmse_err(); - } - - virtual double max_maa_err() { - return 1e-4; - } - - virtual double max_err() { - return max_nmse_err(); - } - - virtual double max_err(ggml_backend_t backend) { - return max_nmse_err(backend); - } - - virtual double err(const float * a, const float * b, size_t n) { - return nmse(a, b, n); - } - - virtual float grad_eps() { - return 1e-1f; - } - - // If false, estimate gradient with 2 points, neglects 3rd order derivative and higher. - // If true, estimate gradient with 4 points, neglects 5th order derivative and higher. - virtual bool grad_precise() { - return false; - } - - // Skip gradient checks if total number of gradients to be checked is larger than this (to speed up the tests). - virtual int64_t grad_nmax() { - return 10000; - } - - // No effect if empty. - // If not empty, skip all gradient checks where the numerical result does not match any of the values. - // Needed for dealing with noncontinuous gradients (e.g. ReLU) where estimation using finite differences is unreliable. - virtual std::vector grad_expect() { - return {}; - } - - virtual void initialize_tensors(ggml_context * ctx) { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t); - } - } - - virtual size_t op_size(ggml_tensor * t) { - size_t size = ggml_nbytes(t); - // add source tensors - for (int i = 0; i < GGML_MAX_SRC; i++) { - if (t->src[i] != NULL) { - size += ggml_nbytes(t->src[i]); - } - } - return size; - } - - virtual uint64_t op_flops(ggml_tensor * t) { - GGML_UNUSED(t); - return 0; - } - - virtual bool run_whole_graph() { return false; } - virtual std::vector fusion_test_nodes() { return {}; } - virtual bool use_weight_context() { return false; } - - ggml_cgraph * gf = nullptr; - ggml_cgraph * gb = nullptr; - - static const int sentinel_size = 1024; - - test_mode mode; - - std::vector sentinels; - - std::string current_op_name; - bool contains_f16 = false; - - // Used by the WebGPU backend to relax error thresholds on ops on f16 tensors - void check_for_f16_tensor(ggml_context * ctx) { - contains_f16 = false; - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_F16) { - contains_f16 = true; - break; - } - } - } - - void add_sentinel(ggml_context * ctx) { - if (mode == MODE_PERF || mode == MODE_GRAD || mode == MODE_SUPPORT) { - return; - } - ggml_tensor * sentinel = ::ggml_new_tensor_1d(ctx, GGML_TYPE_F32, sentinel_size); - ggml_format_name(sentinel, "sent_%zu", sentinels.size()); - sentinels.push_back(sentinel); - } - - // hijack ggml_new_tensor to add sentinels after each tensor to check for overflows in the backend - - ggml_tensor * ggml_new_tensor(ggml_context * ctx, ggml_type type, int n_dims, const int64_t * ne) { - ggml_tensor * t = ::ggml_new_tensor(ctx, type, n_dims, ne); - add_sentinel(ctx); - return t; - } - - ggml_tensor * ggml_new_tensor_1d(ggml_context * ctx, ggml_type type, int64_t ne0) { - ggml_tensor * t = ::ggml_new_tensor_1d(ctx, type, ne0); - add_sentinel(ctx); - return t; - } - - ggml_tensor * ggml_new_tensor_2d(ggml_context * ctx, ggml_type type, int64_t ne0, int64_t ne1) { - ggml_tensor * t = ::ggml_new_tensor_2d(ctx, type, ne0, ne1); - add_sentinel(ctx); - return t; - } - - ggml_tensor * ggml_new_tensor_3d(ggml_context * ctx, ggml_type type, int64_t ne0, int64_t ne1, int64_t ne2) { - ggml_tensor * t = ::ggml_new_tensor_3d(ctx, type, ne0, ne1, ne2); - add_sentinel(ctx); - return t; - } - - ggml_tensor * ggml_new_tensor_4d(ggml_context * ctx, ggml_type type, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3) { - ggml_tensor * t = ::ggml_new_tensor_4d(ctx, type, ne0, ne1, ne2, ne3); - add_sentinel(ctx); - return t; - } - - // Checks an op against the test filter, which is a comma separated list of OP names or specific variations - bool matches_filter(ggml_tensor * op, const char * op_names_filter) { - if (op_names_filter) { - const auto op_name = op_desc(op); - const auto op_full_name = op_name + "(" + vars() + ")"; - std::string_view filter(op_names_filter); - while (!filter.empty()) { - auto comma_pos = filter.find_first_of(','); - const auto lparen_pos = filter.find_first_of('('); - if (lparen_pos < comma_pos) { - auto rparen_pos = filter.find_first_of(')'); - comma_pos = filter.find_first_of(',', rparen_pos); - const auto op_filter = filter.substr(0, comma_pos); - if (op_filter == op_full_name) { - return true; - } - } else { - const auto op_filter = filter.substr(0, comma_pos); - if (op_filter == op_name) { - return true; - } - } - filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : ""; - } - return false; - } else { - return true; - } - } - - test_status_t eval(ggml_backend_t backend1, - ggml_backend_t backend2, - const char * op_names_filter, - printer * output_printer) { - mode = MODE_TEST; - - ggml_init_params params = { - /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead(), - /* .mem_base = */ NULL, - /* .no_alloc = */ true, - }; - const bool use_weights = use_weight_context(); - - ggml_context_ptr ctx(ggml_init(params)); - GGML_ASSERT(ctx); - ggml_context_ptr ctx_weights(use_weights ? ggml_init(params) : nullptr); - GGML_ASSERT(!use_weights || ctx_weights); - - gf = ggml_new_graph(ctx.get()); - - // pre-graph sentinel - add_sentinel(ctx.get()); - if (ctx_weights) { - add_sentinel(ctx_weights.get()); - } - - ggml_tensor * out = build_graph(ctx.get(), ctx_weights.get()); - current_op_name = op_desc(out); - check_for_f16_tensor(ctx.get()); - - if (!matches_filter(out, op_names_filter)) { - //printf(" %s: skipping\n", op_desc(out).c_str()); - return test_status_t::SKIPPED; - } - - // check if the backends support the ops - bool supported = true; - for (ggml_backend_t backend : {backend1, backend2}) { - for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { - if (!ggml_backend_supports_op(backend, t)) { - supported = false; - break; - } - } - } - - if (!supported) { - // Create test result for unsupported operation - test_result result(ggml_backend_name(backend1), current_op_name, vars(), "test", - false, false, "not supported"); - - print_test_result_locked(output_printer, result); - - return test_status_t::NOT_SUPPORTED; - } - - // post-graph sentinel - add_sentinel(ctx.get()); - if (ctx_weights) { - add_sentinel(ctx_weights.get()); - } - - ggml_backend_buffer_ptr buf_weights(nullptr); - if (ctx_weights) { - buf_weights.reset(ggml_backend_alloc_ctx_tensors(ctx_weights.get(), backend1)); - if (buf_weights == NULL) { - printf("failed to allocate weight tensors [%s] ", ggml_backend_name(backend1)); - return test_status_t::FAIL; - } - ggml_backend_buffer_set_usage(buf_weights.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - } - - // allocate - ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend1)); - - if (buf == NULL) { - printf("failed to allocate tensors [%s] ", ggml_backend_name(backend1)); - return test_status_t::FAIL; - } - - // build graph - ggml_build_forward_expand(gf, out); - - // add sentinels as graph nodes so that they are checked in the callback - for (ggml_tensor * sentinel : sentinels) { - ggml_graph_add_node(gf, sentinel); - } - - // randomize tensors - initialize_tensors(ctx.get()); - if (ctx_weights) { - initialize_tensors(ctx_weights.get()); - } - - // compare - struct callback_userdata { - bool ok; - test_case * tc; - ggml_backend_t backend1; - ggml_backend_t backend2; - }; - - callback_userdata ud { - true, - this, - backend1, - backend2, - }; - - auto callback = [](int index, ggml_tensor * t1, ggml_tensor * t2, void * user_data) -> bool { - callback_userdata * ud = (callback_userdata *) user_data; - const char * bn1 = ggml_backend_name(ud->backend1); - const char * bn2 = ggml_backend_name(ud->backend2); - - if (t1->op == GGML_OP_NONE) { - // sentinels must be unchanged - std::vector t1_data(ggml_nbytes(t1)); - std::vector t2_data(ggml_nbytes(t2)); - ggml_backend_tensor_get(t1, t1_data.data(), 0, ggml_nbytes(t1)); - ggml_backend_tensor_get(t2, t2_data.data(), 0, ggml_nbytes(t2)); - - if (memcmp(t1_data.data(), t2_data.data(), ggml_nbytes(t1)) != 0) { - printf("sentinel mismatch: %s ", t1->name); - ud->ok = false; - return true; - } - } - - std::vector f1 = tensor_to_float(t1); - std::vector f2 = tensor_to_float(t2); - - for (size_t i = 0; i < f1.size(); i++) { - // check for nans - if (std::isnan(f1[i]) || std::isnan(f2[i])) { - printf("[%s] NaN at index %zu (%s=%f %s=%f) ", ggml_op_desc(t1), i, bn1, f1[i], bn2, f2[i]); - ud->ok = false; - return true; - } - // check for infs: both must be inf of the same sign, or both must be finite - if (isinf_or_max(f1[i]) || isinf_or_max(f2[i])) { - if (isinf_or_max(f1[i]) && isinf_or_max(f2[i])) { - if (std::signbit(f1[i]) != std::signbit(f2[i])) { - printf("[%s] inf sign mismatch: %s=%f %s=%f ", ggml_op_desc(t1), bn1, f1[i], bn2, f2[i]); - ud->ok = false; - return true; - } - } else { - printf("[%s] inf mismatch: %s=%f %s=%f ", ggml_op_desc(t1), bn1, f1[i], bn2, f2[i]); - ud->ok = false; - return true; - } - } - } - - double err = ud->tc->err(f1.data(), f2.data(), f1.size()); - if (err > ud->tc->max_err(ud->backend1)) { - printf("[%s] ERR = %.9f > %.9f ", ggml_op_desc(t1), err, ud->tc->max_err(ud->backend1)); - //for (int i = 0; i < (int) f1.size(); i++) { - // printf("%5d %9.6f %9.6f, diff = %9.6f\n", i, f1[i], f2[i], f1[i] - f2[i]); - //} - //printf("\n"); - //exit(1); - ud->ok = false; - } - return true; - - GGML_UNUSED(index); - }; - - std::vector fused_nodes_to_verify = fusion_test_nodes(); - if (fused_nodes_to_verify.size() == 0 && run_whole_graph()) { - fused_nodes_to_verify.push_back(out); - } - const bool cmp_ok = ggml_backend_compare_graph_backend(backend1, backend2, gf, callback, &ud, - run_whole_graph() ? fused_nodes_to_verify.data() : nullptr, - fused_nodes_to_verify.size()); - - // Create test result - bool test_passed = ud.ok && cmp_ok; - std::string error_msg = test_passed ? "" : (!cmp_ok ? "compare failed" : "test failed"); - test_result result(ggml_backend_name(backend1), current_op_name, vars(), "test", supported, test_passed, - error_msg); - - print_test_result_locked(output_printer, result); - - return test_passed ? test_status_t::OK : test_status_t::FAIL; - } - - bool eval_perf(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) { - mode = MODE_PERF; - - static const size_t graph_nodes = 8192; - - ggml_init_params params = { - /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead_custom(graph_nodes, false), - /* .mem_base = */ NULL, - /* .no_alloc = */ true, - }; - const bool use_weights = use_weight_context(); - - ggml_context_ptr ctx(ggml_init(params)); // smart ptr - GGML_ASSERT(ctx); - ggml_context_ptr ctx_weights(use_weights ? ggml_init(params) : nullptr); - GGML_ASSERT(!use_weights || ctx_weights); - - ggml_tensor * out = build_graph(ctx.get(), ctx_weights.get()); - current_op_name = op_desc(out); - if (!matches_filter(out, op_names_filter)) { - //printf(" %s: skipping\n", op_desc(out).c_str()); - return true; - } - - if (!ggml_backend_supports_op(backend, out)) { - // Create test result for unsupported performance test - test_result result(ggml_backend_name(backend), current_op_name, vars(), "perf", false, false, - "not supported"); - - output_printer->print_test_result(result); - - return true; - } - - ggml_backend_buffer_ptr buf_weights(nullptr); - if (ctx_weights) { - buf_weights.reset(ggml_backend_alloc_ctx_tensors(ctx_weights.get(), backend)); - if (buf_weights == NULL) { - printf("failed to allocate weight tensors\n"); - return false; - } - ggml_backend_buffer_set_usage(buf_weights.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - } - - // allocate - ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend)); // smart ptr - - if (buf == NULL) { - printf("failed to allocate tensors\n"); - return false; - } - - // randomize tensors - initialize_tensors(ctx.get()); - if (ctx_weights) { - initialize_tensors(ctx_weights.get()); - } - - // build graph - ggml_cgraph * gf = ggml_new_graph_custom(ctx.get(), graph_nodes, false); - ggml_build_forward_expand(gf, out); - - // warmup run - ggml_status status = ggml_backend_graph_compute(backend, gf); - if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); - return false; - } - - // determine number of runs - int n_runs; - bool is_cpu = ggml_backend_dev_type(ggml_backend_get_device(backend)) == GGML_BACKEND_DEVICE_TYPE_CPU; - if (op_flops(out) > 0) { - // based on flops - const uint64_t GFLOP = 1000 * 1000 * 1000; - const uint64_t target_flops_cpu = 8ULL * GFLOP; - const uint64_t target_flops_gpu = 100ULL * GFLOP; - uint64_t target_flops = is_cpu ? target_flops_cpu : target_flops_gpu; - n_runs = (int)std::min(ggml_graph_size(gf) - ggml_graph_n_nodes(gf), target_flops / op_flops(out)) + 1; - } else { - // based on memory size - const size_t GB = 1ULL << 30; - const size_t target_size_cpu = 8 * GB; - const size_t target_size_gpu = 32 * GB; - size_t target_size = is_cpu ? target_size_cpu : target_size_gpu; - n_runs = (int)std::min(ggml_graph_size(gf) - ggml_graph_n_nodes(gf), target_size / op_size(out)) + 1; - } - - // duplicate the op - for (int i = 1; i < n_runs; i++) { - ggml_graph_add_node(gf, out); - } - - // calculate memory - size_t mem = n_runs * op_size(out); - auto tensor_op_size = [](ggml_tensor * t) { - size_t size = ggml_nbytes(t); - // add source tensors - for (int i = 0; i < GGML_MAX_SRC; i++) { - if (t->src[i] != NULL) { - size += ggml_nbytes(t->src[i]); - } - } - return size; - }; - for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) { - if (ggml_is_view_op(ggml_graph_node(gf, i)->op) || ggml_graph_node(gf, i) == out) { - continue; - } - mem += tensor_op_size(ggml_graph_node(gf, i)); - } - - // run - int64_t total_time_us = 0; - int64_t total_mem = 0; - int total_runs = 0; - do { - int64_t start_time = ggml_time_us(); - ggml_status status = ggml_backend_graph_compute(backend, gf); - if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); - return false; - } - int64_t end_time = ggml_time_us(); - - total_time_us += end_time - start_time; - total_mem += mem; - total_runs += n_runs; - } while (total_time_us < 1000*1000); // run for at least 1 second - - // Create test result - double avg_time_us = (double) total_time_us / total_runs; - double calculated_flops = (op_flops(out) > 0) ? (op_flops(out) * total_runs) / (total_time_us / 1e6) : 0.0; - double calculated_bandwidth = - (op_flops(out) == 0) ? total_mem / (total_time_us / 1e6) / 1024.0 / 1024.0 / 1024.0 : 0.0; - size_t calculated_memory_kb = op_size(out) / 1024; - - test_result result(ggml_backend_name(backend), current_op_name, vars(), "perf", true, true, "", avg_time_us, - calculated_flops, calculated_bandwidth, calculated_memory_kb, total_runs); - - if (output_printer) { - output_printer->print_test_result(result); - } - - return true; - } - - bool eval_support(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) { - mode = MODE_SUPPORT; - - static const size_t graph_nodes = 8192; - - ggml_init_params params = { - /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead_custom(graph_nodes, false), - /* .mem_base = */ NULL, - /* .no_alloc = */ true, - }; - ggml_context_ptr ctx(ggml_init(params)); // smart ptr - GGML_ASSERT(ctx); - - gf = ggml_new_graph_custom(ctx.get(), graph_nodes, false); - - ggml_tensor * out = build_graph(ctx.get()); - current_op_name = op_desc(out); - - if (!matches_filter(out, op_names_filter)) { - return true; - } - - bool supported = ggml_backend_supports_op(backend, out); - - std::string device_desc = ggml_backend_dev_description(ggml_backend_get_device(backend)); - std::string backend_reg_name = ggml_backend_reg_name(ggml_backend_dev_backend_reg(ggml_backend_get_device(backend))); - - test_result result(ggml_backend_name(backend), current_op_name, vars(), "support", supported, supported, - supported ? "yes" : "no", 0.0, 0.0, 0.0, 0, 0, device_desc, backend_reg_name); - - output_printer->print_test_result(result); - - return true; - } - - bool eval_grad(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) { - mode = MODE_GRAD; - const std::vector expect = grad_expect(); - - ggml_init_params params = { - /* .mem_size = */ ggml_tensor_overhead()*128 + 2*ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE, true), - /* .mem_base = */ NULL, - /* .no_alloc = */ true, - }; - ggml_context_ptr ctx(ggml_init(params)); // smart ptr - GGML_ASSERT(ctx); - - gf = ggml_new_graph_custom(ctx.get(), GGML_DEFAULT_GRAPH_SIZE, true); - gb = ggml_new_graph_custom(ctx.get(), GGML_DEFAULT_GRAPH_SIZE, true); - - ggml_tensor * out = build_graph(ctx.get()); - - if (!matches_filter(out, op_names_filter) || out->op == GGML_OP_OPT_STEP_ADAMW) { - return true; - } - - if (out->type != GGML_TYPE_F32) { - output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), - test_status_t::NOT_SUPPORTED, - out->name + std::string("->type != FP32"))); - return true; - } - - // Print operation info first - output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend))); - - // check if the backend supports the ops - bool supported = true; - bool any_params = false; - std::string failure_reason; - - for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { - if (!ggml_backend_supports_op(backend, t)) { - supported = false; - failure_reason = ggml_backend_name(backend); - break; - } - if ((t->flags & GGML_TENSOR_FLAG_PARAM)) { - any_params = true; - if (t->type != GGML_TYPE_F32) { - supported = false; - failure_reason = std::string(t->name) + "->type != FP32"; - break; - } - } - } - if (!any_params) { - supported = false; - failure_reason = op_desc(out); - } - - if (!supported) { - output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), - test_status_t::NOT_SUPPORTED, failure_reason)); - return true; - } - - int64_t ngrads = 0; - for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { - if (t->flags & GGML_TENSOR_FLAG_PARAM) { - ngrads += ggml_nelements(t); - } - } - if (ngrads > grad_nmax()) { - test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); - info.set_large_tensor_skip(); - output_printer->print_operation(info); - return true; - } - - - if (!ggml_is_scalar(out)) { - out = ggml_sum(ctx.get(), out); - ggml_set_name(out, "sum_of_out"); - } - ggml_set_loss(out); - - ggml_build_forward_expand(gf, out); - ggml_graph_cpy(gf, gb); - ggml_build_backward_expand(ctx.get(), gb, nullptr); - if (expect.size() != 1 || expect[0] != 0.0f) { - GGML_ASSERT(ggml_graph_n_nodes(gb) > ggml_graph_n_nodes(gf)); - for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { - GGML_ASSERT(!(t->flags & GGML_TENSOR_FLAG_PARAM) || ggml_graph_get_grad(gb, t)->op != GGML_OP_NONE); - } - } - - for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { - if (!ggml_backend_supports_op(backend, t)) { - output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), - test_status_t::NOT_SUPPORTED, - ggml_backend_name(backend))); - supported = false; - break; - } - if ((t->flags & GGML_TENSOR_FLAG_PARAM) && t->type != GGML_TYPE_F32) { - output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), - test_status_t::NOT_SUPPORTED, - std::string(t->name) + "->type != FP32")); - supported = false; - break; - } - } - if (!supported) { - return true; - } - - // allocate - ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend)); // smart ptr - if (buf == NULL) { - test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); - info.set_error("allocation", ""); - output_printer->print_operation(info); - return false; - } - - initialize_tensors(ctx.get()); // Randomizes all tensors (including gradients). - ggml_graph_reset(gb); // Sets gradients to 1 if loss, 0 otherwise. - - ggml_status status = ggml_backend_graph_compute(backend, gf); - if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); - return false; - } - status = ggml_backend_graph_compute(backend, gb); - if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); - return false; - } - - bool ok = true; - for (struct ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) { - if (!(t->flags & GGML_TENSOR_FLAG_PARAM)) { - continue; - } - - const char * bn = ggml_backend_name(backend); - const int64_t ne = ggml_nelements(t); - - std::vector ga; - struct ggml_tensor * grad = ggml_graph_get_grad(gb, t); - if (grad) { - ga = tensor_to_float(grad); - } else { - ga.resize(ne); // default value is 0.0f - } - - for (int64_t i = 0; i < ne; ++i) { // gradient algebraic - // check for nans - if (!std::isfinite(ga[i])) { - test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); - info.set_gradient_info(i, bn, ga[i]); - output_printer->print_operation(info); - ok = false; - break; - } - } - if (!ok) { - break; - } - - std::vector gn(ne); // gradient numeric - GGML_ASSERT(ga.size() == gn.size()); - - std::vector x0 = tensor_to_float(t); // original t data - GGML_ASSERT(ggml_is_scalar(out)); - GGML_ASSERT(out->type == GGML_TYPE_F32); - - const float eps = grad_eps(); - for (int64_t i = 0; i < ne; ++i) { - const float xiu = x0[i] + 1.0f*eps; // x, index i, up - const float xiuh = x0[i] + 0.5f*eps; // x, index i, up half - const float xidh = x0[i] - 0.5f*eps; // x, index i, down half - const float xid = x0[i] - 1.0f*eps; // x, index i, down - - float fu, fuh, fdh, fd; // output values for xiu, xiuh, xid, xidh - - ggml_backend_tensor_set(t, &xiu, i*sizeof(float), sizeof(float)); - status = ggml_backend_graph_compute(backend, gf); - if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); - return false; - } - ggml_backend_tensor_get(out, &fu, 0, ggml_nbytes(out)); - - ggml_backend_tensor_set(t, &xid, i*sizeof(float), sizeof(float)); - status = ggml_backend_graph_compute(backend, gf); - if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); - return false; - } - ggml_backend_tensor_get(out, &fd, 0, ggml_nbytes(out)); - - if (grad_precise()) { - ggml_backend_tensor_set(t, &xiuh, i*sizeof(float), sizeof(float)); - status = ggml_backend_graph_compute(backend, gf); - if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); - return false; - } - ggml_backend_tensor_get(out, &fuh, 0, ggml_nbytes(out)); - - ggml_backend_tensor_set(t, &xidh, i*sizeof(float), sizeof(float)); - status = ggml_backend_graph_compute(backend, gf); - if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); - return false; - } - ggml_backend_tensor_get(out, &fdh, 0, ggml_nbytes(out)); - - gn[i] = (8.0*(double)fuh + (double)fd - (8.0*(double)fdh + (double)fu)) / (6.0*(double)eps); - } else { - gn[i] = (fu - fd) / (2.0f*eps); - } - - ggml_backend_tensor_set(t, x0.data(), 0, ggml_nbytes(t)); - } - - const double err = mean_abs_asymm(gn.data(), ga.data(), gn.size(), expect); - if (err > max_maa_err()) { - test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); - info.set_maa_error(err, max_maa_err()); - output_printer->print_operation(info); - ok = false; - break; - } - if (!ok) { - break; - } - } - - // Create final test result - test_operation_info final_info(op_desc(out), vars(), ggml_backend_name(backend)); - if (!ok) { - final_info.set_compare_failure(); - } - final_info.status = ok ? test_status_t::OK : test_status_t::FAIL; - output_printer->print_operation(final_info); - - if (ok) { - return true; - } - - return false; - } -}; - - -// #################################### -// ## Section 2: GGML Op Definitions ## -// #################################### - - -// The following is an example showing the bare minimum for creating a test for a GGML op. - -// GGML_OP_EXAMPLE -struct test_example : public test_case { - // Always define these 2 or variants thereof: - const ggml_type type; // The type of the input tensors. - const std::array ne; // The shape of the input tensors. - // For some ops it's necessary to define multiple types or shapes for the inputs. - // Or they may need additional parameters. - - // Put all parameters needed to fully define the test into one of the VARS_TO_STR macros. - // In most cases these are just the properties of the struct that you defined above. - // This is needed for info prints. - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - // Define a constructor for the struct. - // In most cases it will be sufficient to have the same arguments as the struct has properties - // and just use initializer lists. - test_example(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}) - : type(type), ne(ne) {} - - // Define how a simple GGML compute graph can be constructed for the new GGML op. - ggml_tensor * build_graph(ggml_context * ctx) override { - // Step 1: create input tensors that don't depend on any other tensors: - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); // Setting names is optional but it's useful for debugging. - - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(b, "b"); - - // Step 2: use the op that you want to test in the GGML compute graph. - ggml_tensor * out = ggml_add(ctx, a, b); // For this example we're just doing a simple addition. - ggml_set_name(out, "out"); - - // Step 3: return the output tensor. - return out; - } - // In order to also check the gradients for your op, add calls like ggml_set_param(a) - // immediately after you create the tensors. - // This is optional and only makes sense if a backward pass has actually been implemented for the new op. -}; - - -// GGML_OP_UNARY -struct test_unary : public test_case { - const ggml_unary_op op; - const ggml_type type; - const std::array ne_a; - int v; // view (1 : non-contiguous a) - - std::string vars() override { - return VARS_TO_STR3(type, ne_a, v); - } - - test_unary(ggml_unary_op op, - ggml_type type = GGML_TYPE_F32, - std::array ne_a = {128, 2, 2, 2}, - int v = 0) - : op(op), type(type), ne_a(ne_a), v(v) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - const bool grad_supported = op == GGML_UNARY_OP_ABS || op == GGML_UNARY_OP_SGN || op == GGML_UNARY_OP_NEG || - op == GGML_UNARY_OP_STEP || op == GGML_UNARY_OP_RELU || op == GGML_UNARY_OP_SILU || - op == GGML_UNARY_OP_EXPM1 || op == GGML_UNARY_OP_SOFTPLUS; - - ggml_tensor * a; - if (v & 1) { - auto ne = ne_a; - ne[0] *= 3; - ne[1] *= 2; - ne[2] *= 5; - ne[3] *= 4; - a = ggml_new_tensor(ctx, type, 4, ne.data()); - if (grad_supported) { - ggml_set_param(a); - } - ggml_set_name(a, "a"); - - a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); - ggml_set_name(a, "view_of_a"); - } else { - a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - if (grad_supported) { - ggml_set_param(a); - } - ggml_set_name(a, "a"); - } - - ggml_tensor * out = ggml_unary(ctx, a, op); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - float min = -150.f; - float max = 150.f; - - // Keep FP16 exp/expm1 inputs in-range so all backends stay finite instead of - // disagreeing on whether overflow saturates to max-F16 or produces +inf. - if (type == GGML_TYPE_F16 && (op == GGML_UNARY_OP_EXP || op == GGML_UNARY_OP_EXPM1)) { - min = -10.f; - max = 10.f; - } - - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - // test extended range of values to check for NaNs in GELU - init_tensor_uniform(t, min, max); - } - } - - float grad_eps() override { - return 15.0f; - } - - std::vector grad_expect() override { - if (op == GGML_UNARY_OP_ABS) { - return {-1.0f, 1.0f}; - } - if (op == GGML_UNARY_OP_SGN || op == GGML_UNARY_OP_STEP) { - return {0.0f}; - } - if (op == GGML_UNARY_OP_RELU) { - return {0.0f, 1.0f}; - } - return {}; - } - -}; - -// GGML_OP_GLU -struct test_glu : public test_case { - const ggml_glu_op op; - const ggml_type type; - const std::array ne_a; - int v; // view (1 : non-contiguous a) - bool swapped; - - std::string vars() override { - return VARS_TO_STR4(type, ne_a, v, swapped); - } - - test_glu(ggml_glu_op op, - ggml_type type = GGML_TYPE_F32, - std::array ne_a = {128, 2, 2, 2}, - int v = 0, - bool swapped = false) - : op(op), type(type), ne_a(ne_a), v(v), swapped(swapped) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a; - if (v & 1) { - auto ne = ne_a; ne[0] *= 3; - a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); - ggml_set_name(a, "view_of_a"); - } else { - a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_set_name(a, "a"); - } - - ggml_tensor * out = ggml_glu(ctx, a, op, swapped); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - // test extended range of values to check for NaNs in GELU - init_tensor_uniform(t, -150.f, 150.f); - } - } -}; - -struct test_glu_split : public test_case { - const ggml_glu_op op; - const ggml_type type; - const std::array ne_a; - int v; // view (1 : non-contiguous a) - - std::string vars() override { - return VARS_TO_STR3(type, ne_a, v) + ",split"; - } - - test_glu_split(ggml_glu_op op, - ggml_type type = GGML_TYPE_F32, - std::array ne_a = {128, 2, 2, 2}, - int v = 0) - : op(op), type(type), ne_a(ne_a), v(v) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a; - ggml_tensor * b; - if (v & 1) { - auto ne = ne_a; ne[0] *= 3; - a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); - ggml_set_name(a, "view_of_a"); - - b = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(b); - ggml_set_name(b, "b"); - - b = ggml_view_4d(ctx, b, ne_a[0], ne_a[1], ne_a[2], ne_a[3], b->nb[1], b->nb[2], b->nb[3], 0); - ggml_set_name(a, "view_of_b"); - } else { - a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - b = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_set_param(b); - ggml_set_name(b, "b"); - } - - ggml_tensor * out = ggml_glu_split(ctx, a, b, op); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - // test extended range of values to check for NaNs in GELU - init_tensor_uniform(t, -150.f, 150.f); - } - } -}; - -struct test_swiglu_oai : public test_case { - const ggml_type type; - const std::array ne_a; - int v; // view (1 : non-contiguous a) - float alpha; - float limit; - - std::string vars() override { - return VARS_TO_STR5(type, ne_a, v, alpha, limit); - } - - test_swiglu_oai(ggml_type type = GGML_TYPE_F32, - std::array ne_a = {128, 2, 2, 2}, - int v = 0, - float alpha = 1.702f, - float limit = 7.0f) - : type(type), ne_a(ne_a), v(v), alpha(alpha), limit(limit) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a; - ggml_tensor * b; - if (v & 1) { - auto ne = ne_a; ne[0] *= 3; - a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); - ggml_set_name(a, "view_of_a"); - - b = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(b); - ggml_set_name(b, "b"); - - b = ggml_view_4d(ctx, b, ne_a[0], ne_a[1], ne_a[2], ne_a[3], b->nb[1], b->nb[2], b->nb[3], 0); - ggml_set_name(a, "view_of_b"); - } else { - a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - b = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_set_param(b); - ggml_set_name(b, "b"); - } - - ggml_tensor * out = ggml_swiglu_oai(ctx, a, b, alpha, limit); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - // test extended range of values to check for NaNs in GELU - init_tensor_uniform(t, -150.f, 150.f); - } - } -}; - -// GGML_OP_GET_ROWS -struct test_get_rows : public test_case { - const ggml_type type; - const int n; // cols - const int m; // rows - const int r; // rows to get - const int be1; // batch size - const int be2; // batch size - const bool v; // view (non-contiguous src1) - - std::string vars() override { - return VARS_TO_STR7(type, n, m, r, be1, be2, v); - } - - test_get_rows(ggml_type type = GGML_TYPE_F32, int n = 10, int m = 5, int r = 3, int be1 = 1, int be2 = 1, bool v = false) - : type(type), n(n), m(m), r(r), be1(be1), be2(be2), v(v) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * in = ggml_new_tensor_4d(ctx, type, n, m, be1, be2); - ggml_set_name(in, "in"); - - ggml_tensor * rows = ggml_new_tensor_3d(ctx, GGML_TYPE_I32, r, be1, be2); - ggml_set_name(rows, "rows"); - if (v) { - rows = ggml_view_3d(ctx, rows, r/2, be1, be2, rows->nb[1], rows->nb[2], 0); - ggml_set_name(rows, "view_of_rows"); - } - - const bool grad_supported = ggml_is_matrix(in) && ggml_is_vector(rows); - if (grad_supported) { - ggml_set_param(in); - // rows is a constant input -> no gradients - } - - ggml_tensor * out = ggml_get_rows(ctx, in, rows); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I32) { - if (ggml_is_view_op(t->op)) { continue; } - // rows - std::vector data(r*be1*be2); - for (int i = 0; i < r*be1*be2; i++) { - data[i] = rand() % m; - } - ggml_backend_tensor_set(t, data.data(), 0, r * be1 * be2 * sizeof(int)); - } else { - init_tensor_uniform(t); - } - } - } -}; - -// GGML_OP_GET_ROWS_BACK -struct test_get_rows_back : public test_case { - const ggml_type type; - const int n; // cols - const int m; // rows - const int r; // rows to get - const int b; // batch size - const bool v; // view (non-contiguous src1) - - std::string vars() override { - return VARS_TO_STR6(type, n, m, r, b, v); - } - - test_get_rows_back(ggml_type type = GGML_TYPE_F32, int n = 10, int m = 5, int r = 3, int b = 1, bool v = false) - : type(type), n(n), m(m), r(r), b(b), v(v) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * in_forward = ggml_new_tensor_3d(ctx, type, n, m, b); - ggml_set_name(in_forward, "in_forward"); - - ggml_tensor * rows = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, r, b); - ggml_set_name(rows, "rows"); - if (v) { - rows = ggml_view_2d(ctx, rows, r/2, b, rows->nb[1], 0); - ggml_set_name(rows, "view_of_rows"); - } - - ggml_tensor * grad = ggml_new_tensor_3d(ctx, type, n, r, b); - ggml_set_name(grad, "grad"); - - ggml_tensor * out = ggml_get_rows_back(ctx, grad, rows, in_forward); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I32) { - if (ggml_is_view_op(t->op)) { continue; } - // rows - std::vector data(r*b); - for (int i = 0; i < r*b; i++) { - data[i] = rand() % m; - } - ggml_backend_tensor_set(t, data.data(), 0, r * b * sizeof(int)); - } else { - init_tensor_uniform(t); - } - } - } -}; - -static void init_set_rows_row_ids(ggml_tensor * t, int num_rows) { - std::random_device rd; - std::default_random_engine rng(rd()); - for (int i2 = 0; i2 < t->ne[2]; i2++) { - for (int i1 = 0; i1 < t->ne[1]; i1++) { - // generate a shuffled subset of row indices - std::vector data(num_rows); - for (int i = 0; i < num_rows; i++) { - data[i] = i; - } - std::shuffle(data.begin(), data.end(), rng); - data.resize(t->ne[0]); - - const size_t offs = i1*t->nb[1] + i2*t->nb[2]; - if (t->type == GGML_TYPE_I32) { - // TODO: Make a template or something - std::vector data_i32(t->ne[0]); - for (int i = 0; i < t->ne[0]; i++) { - data_i32[i] = static_cast(data[i]); - } - ggml_backend_tensor_set(t, data_i32.data(), offs, t->ne[0]*sizeof(int32_t)); - } else { - ggml_backend_tensor_set(t, data.data(), offs, t->ne[0]*sizeof(int64_t)); - } - } - } -} - -// GGML_OP_SET_ROWS -struct test_set_rows : public test_case { - const ggml_type type_src; - const ggml_type type_dst; - const ggml_type type_idx; - const std::array ne; - const std::array nr23; // broadcast only dims 2 and 3 - const int r; // rows to set - const bool v; // view (non-contiguous src1) - - std::string vars() override { - return VARS_TO_STR7(type_src, type_dst, type_idx, ne, nr23, r, v); - } - - test_set_rows(ggml_type type_src, - ggml_type type_dst, - ggml_type type_idx, - std::array ne, - std::array nr23, - int r, bool v = false) - : type_src(type_src), type_dst(type_dst), type_idx(type_idx), ne(ne), nr23(nr23), r(r), v(v) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * dst = ggml_new_tensor_4d(ctx, type_dst, ne[0], ne[1], ne[2]*nr23[0], ne[3]*nr23[1]); - ggml_set_name(dst, "dst"); - - ggml_tensor * src = ggml_new_tensor_4d(ctx, type_src, ne[0], r, ne[2]*nr23[0], ne[3]*nr23[1]); - ggml_set_name(src, "src"); - - ggml_tensor * row_idxs = ggml_new_tensor_3d(ctx, type_idx, r, ne[2], ne[3]); - ggml_set_name(row_idxs, "row_idxs"); - - if (v) { - src = ggml_view_4d(ctx, src, ne[0], r/2, ne[2]*nr23[0], ne[3]*nr23[1], src->nb[1], src->nb[2], src->nb[3], 0); - row_idxs = ggml_view_3d(ctx, row_idxs, r/2, ne[2], ne[3], row_idxs->nb[1], row_idxs->nb[2], 0); - ggml_set_name(row_idxs, "view_of_rows"); - } - - ggml_tensor * out = ggml_set_rows(ctx, dst, src, row_idxs); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I64 || t->type == GGML_TYPE_I32) { - if (ggml_is_view_op(t->op)) { - continue; - } - - init_set_rows_row_ids(t, ne[1]); - } else { - init_tensor_uniform(t); - } - } - } - - double max_nmse_err() override { - if (type_dst == GGML_TYPE_Q4_0 || type_dst == GGML_TYPE_Q4_1 || type_dst == GGML_TYPE_IQ4_NL || - type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1 || type_dst == GGML_TYPE_Q8_0) { - // estimate what the max nmse error would be if one quantized value is - // off by one. The test values are distributed in [-1,1], so it'll be - // roughly (2.0 / 2^bits)^2, divided by the mean square value of the reference, - // which is roughly 0.25 times the number of elements. - double err_estimate = 1.0f/8.0f; - if (type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1) { - err_estimate /= 2.0f; - } - if (type_dst == GGML_TYPE_Q8_0) { - err_estimate /= 8.0f; - } - err_estimate *= err_estimate; - err_estimate /= 0.25f*float(ne[0] * r * ne[2]*nr23[0] * ne[3]*nr23[1]); - return err_estimate; - } - return 1e-7; - } - - // See dicussion here: https://github.com/ggml-org/llama.cpp/pull/23760#issuecomment-4566312209 - double max_nmse_err(ggml_backend_t backend) override { - ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); - if (type_dst == GGML_TYPE_Q8_0 && strcmp(ggml_backend_reg_name(reg), "WebGPU") == 0) { - return std::max(test_case::max_nmse_err(backend), 2e-7); - } - return test_case::max_nmse_err(backend); - } -}; - -// GGML_OP_ROPE + GGML_OP_VIEW + GGML_OP_SET_ROWS -struct test_rope_set_rows : public test_case { - const ggml_type type; - const ggml_type type_idx; - const std::array ne_a; - int mode; - const int n_ctx{512}; - const int n_dims{128}; - - std::string vars() override { - return VARS_TO_STR4(type, type_idx, ne_a, mode); - } - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "ROPE_SET_ROWS"; - } - - bool run_whole_graph() override { return true; } - - test_rope_set_rows(ggml_type type, - ggml_type type_idx, - std::array ne_a, - int mode) - : type(type), type_idx(type_idx), ne_a(ne_a), mode(mode) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne_a[0], ne_a[1], ne_a[2], 1); - ggml_set_name(a, "a"); - - const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; - const bool is_vision = mode == GGML_ROPE_TYPE_VISION; - - ggml_tensor * pos; - if (is_mrope || is_vision) { - pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne_a[2] * 4); - } else { - pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne_a[2]); - } - ggml_set_name(pos, "pos"); - - float fs = 1.4245f; - float ef = 0.7465f; - float af = 1.4245f; - ggml_tensor * freq = nullptr; - - ggml_tensor * rope = nullptr; - if (is_mrope) { - if (is_vision) { - GGML_ASSERT(n_dims/4 > 0); - int rope_sections[4] = {n_dims/4, n_dims/4, 0, 0}; // Vision-RoPE only use first two dimension for image (x, y) coordinate - rope = ggml_rope_multi(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); - } else { - GGML_ASSERT(n_dims/3 > 0); - int rope_sections[4] = {n_dims/3, n_dims/3, n_dims/3, 0}; - rope = ggml_rope_multi(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); - } - } else { - rope = ggml_rope(ctx, a, pos, ne_a[0], mode); - } - - ggml_tensor * view = ggml_view_2d(ctx, rope, ne_a[0] * ne_a[1], ne_a[2], rope->nb[2], 0); - - ggml_tensor * dst = ggml_new_tensor_4d(ctx, type, ne_a[0] * ne_a[1], ne_a[2] * ne_a[3], 1, 1); - ggml_set_name(dst, "dst"); - - ggml_tensor * row_idxs = ggml_new_tensor_3d(ctx, type_idx, ne_a[2], 1, 1); - ggml_set_name(row_idxs, "row_idxs"); - - ggml_tensor * out = ggml_set_rows(ctx, dst, view, row_idxs); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (strcmp(t->name, "row_idxs") == 0) { - if (ggml_is_view_op(t->op)) { - continue; - } - init_set_rows_row_ids(t, ne_a[2]); - } else if (t->type == GGML_TYPE_I32) { - // pos - const int num_pos_ids = (mode & GGML_ROPE_TYPE_MROPE) ? ne_a[2] * 4 : ne_a[2]; - std::vector data(num_pos_ids); - for (int i = 0; i < num_pos_ids; i++) { - data[i] = rand() % n_ctx; - } - ggml_backend_tensor_set(t, data.data(), 0, num_pos_ids * sizeof(int)); - } else { - if (t->ne[0] == n_dims/2) { - // frequency factors in the range [0.9f, 1.1f] - init_tensor_uniform(t, 0.9f, 1.1f); - } else { - init_tensor_uniform(t); - } - } - } - } -}; - -// GGML_OP_RMS_NORM + GGML_OP_MUL + GGML_OP_ROPE (+ GGML_OP_VIEW + GGML_OP_SET_ROWS) -struct test_rms_norm_mul_rope : public test_case { - const std::array ne; - const float eps; - const bool multi_add; // test a sequence of adds feeding into rms_norm - const bool set_rows; - int mode; - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "RMS_NORM_MUL_ROPE"; - } - - bool run_whole_graph() override { return true; } - - std::string vars() override { - return VARS_TO_STR5(ne, eps, multi_add, set_rows, mode); - } - - test_rms_norm_mul_rope(std::array ne, float eps = 1e-6f, bool multi_add = false, - bool set_rows = false, int mode = GGML_ROPE_TYPE_NORMAL) - : ne(ne), eps(eps), multi_add(multi_add), set_rows(set_rows), mode(mode) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], 1); - ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], 1); - ggml_tensor * c = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], 1); - - if (multi_add) { - a = ggml_add(ctx, ggml_add(ctx, a, b), c); - } - - a = ggml_mul(ctx, ggml_rms_norm(ctx, a, eps), b); - - ggml_tensor * pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne[2]); - - ggml_tensor * rope = ggml_rope(ctx, a, pos, ne[0], mode); - - ggml_tensor * out; - - if (set_rows) { - ggml_tensor * view = ggml_view_2d(ctx, rope, ne[0] * ne[1], ne[2], rope->nb[2], 0); - - ggml_tensor * dst = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, ne[0] * ne[1], ne[2] * ne[3], 1, 1); - ggml_set_name(dst, "dst"); - - ggml_tensor * row_idxs = ggml_new_tensor_3d(ctx, GGML_TYPE_I64, ne[2], 1, 1); - ggml_set_name(row_idxs, "row_idxs"); - - out = ggml_set_rows(ctx, dst, view, row_idxs); - ggml_set_name(out, "out"); - } else { - out = rope; - } - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I64 || t->type == GGML_TYPE_I32) { - if (ggml_is_view_op(t->op)) { - continue; - } - - init_set_rows_row_ids(t, ne[2]); - } else { - init_tensor_uniform(t); - } - } - } -}; - -// GGML_OP_ARGMAX -struct test_argmax : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_argmax(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 100, 1, 1}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_argmax(ctx, a); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - std::random_device rd; - std::default_random_engine rng(rd()); - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_F32) { - // initialize with unique values to avoid ties - for (int64_t r = 0; r < ggml_nrows(t); r++) { - std::vector data(t->ne[0]); - for (int i = 0; i < t->ne[0]; i++) { - data[i] = i; - } - std::shuffle(data.begin(), data.end(), rng); - ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(float)); - } - } else { - init_tensor_uniform(t); - } - } - } - - double max_nmse_err() override { - return 0.0; - } -}; - -// GGML_OP_COUNT_EQUAL -struct test_count_equal : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_count_equal(ggml_type type = GGML_TYPE_F32, - std::array ne = {4, 500, 1, 1}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - ggml_tensor * a_argmax = ggml_argmax(ctx, a); - ggml_set_name(a_argmax, "a_argmax"); - - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(b, "b"); - - ggml_tensor * b_argmax = ggml_argmax(ctx, b); - ggml_set_name(b_argmax, "b_argmax"); - - ggml_tensor * out = ggml_count_equal(ctx, a_argmax, b_argmax); - ggml_set_name(out, "out"); - - return out; - } - - double max_nmse_err() override { - return 0.0; - } - - void initialize_tensors(ggml_context * ctx) override { - std::random_device rd; - std::default_random_engine rng(rd()); - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_F32) { - // initialize with unique values to avoid ties - for (int64_t r = 0; r < ggml_nrows(t); r++) { - std::vector data(t->ne[0]); - for (int i = 0; i < t->ne[0]; i++) { - data[i] = i; - } - std::shuffle(data.begin(), data.end(), rng); - ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(float)); - } - } else { - init_tensor_uniform(t); - } - } - } -}; - -// GGML_OP_REPEAT -struct test_repeat : public test_case { - const ggml_type type; - const std::array ne; - const std::array nr; - - std::string vars() override { - return VARS_TO_STR3(type, ne, nr); - } - - size_t op_size(ggml_tensor * t) override { - return ggml_nbytes(t) * 2; - } - - test_repeat(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}, - std::array nr = {2, 2, 2, 2}) - : type(type), ne(ne), nr(nr) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * target = ggml_new_tensor_4d(ctx, type, ne[0]*nr[0], ne[1]*nr[1], ne[2]*nr[2], ne[3]*nr[3]); - ggml_set_name(target, "target"); - - ggml_tensor * src = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(src); - ggml_set_name(src, "src"); - - ggml_tensor * out = ggml_repeat(ctx, src, target); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_REPEAT_BACK -struct test_repeat_back : public test_case { - const ggml_type type; - const std::array ne; - const std::array nr; - const bool v; // whether src is a noncontiguous view - - std::string vars() override { - return VARS_TO_STR4(type, ne, nr, v); - } - - size_t op_size(ggml_tensor * t) override { - return ggml_nbytes(t) * 2; - } - - test_repeat_back(ggml_type type = GGML_TYPE_F32, - std::array ne = {8, 6, 4, 2}, - std::array nr = {2, 2, 2, 2}, - bool v = false) - : type(type), ne(ne), nr(nr), v(v) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * src = ggml_new_tensor_4d(ctx, type, ne[0]*nr[0], ne[1]*nr[1], ne[2]*nr[2], ne[3]*nr[3]); - ggml_set_name(src, "src"); - - if (v) { - GGML_ASSERT(ne[0] % 2 == 0); - GGML_ASSERT(ne[1] % 2 == 0); - GGML_ASSERT(ne[2] % 2 == 0); - GGML_ASSERT(ne[3] % 2 == 0); - GGML_ASSERT(nr[0] % 2 == 0 || nr[0] == 1); - GGML_ASSERT(nr[1] % 2 == 0 || nr[1] == 1); - GGML_ASSERT(nr[2] % 2 == 0 || nr[2] == 1); - GGML_ASSERT(nr[3] % 2 == 0 || nr[3] == 1); - - const int64_t ne00 = nr[0] == 1 ? src->ne[0] : src->ne[0] / 2; - const int64_t ne01 = nr[1] == 1 ? src->ne[1] : src->ne[1] / 2; - const int64_t ne02 = nr[2] == 1 ? src->ne[2] : src->ne[2] / 2; - const int64_t ne03 = nr[3] == 1 ? src->ne[3] : src->ne[3] / 2; - - src = ggml_view_4d(ctx, src, ne00, ne01, ne02, ne03, src->nb[1], src->nb[2], src->nb[3], 0); - } - - ggml_tensor * target = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(target, "target"); - - ggml_tensor * out = ggml_repeat_back(ctx, src, target); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_DUP -struct test_dup : public test_case { - const ggml_type type; - const std::array ne; - const std::array permute; - bool _use_permute; - - std::string vars() override { - std::string v = VARS_TO_STR2(type, ne); - if (_use_permute) v += "," + VAR_TO_STR(permute); - return v; - } - - test_dup(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 10, 20, 1}, - std::array permute = {0, 0, 0, 0}) - : type(type), ne(ne), permute(permute), - _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * src = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(src); - ggml_set_name(src, "src"); - - if (_use_permute) { - src = ggml_permute(ctx, src, permute[0], permute[1], permute[2], permute[3]); - ggml_set_name(src, "src_permuted"); - } - - ggml_tensor * out = ggml_dup(ctx, src); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_SET -struct test_set : public test_case { - const ggml_type type_src; - const ggml_type type_dst; - const std::array ne; - const int dim; - const bool inplace; - - std::string vars() override { - return VARS_TO_STR5(type_src, type_dst, ne, dim, inplace); - } - - size_t op_size(ggml_tensor * t) override { - return ggml_nbytes(t) + ggml_nbytes(t->src[0]); - } - - test_set(ggml_type type_src = GGML_TYPE_F32, ggml_type type_dst = GGML_TYPE_F32, - std::array ne = {6, 5, 4, 3}, int dim = 1, bool inplace = false) - : type_src(type_src), type_dst(type_dst), ne(ne), dim(dim), inplace(inplace) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * src = ggml_new_tensor(ctx, type_src, 4, ne.data()); - ggml_set_param(src); - ggml_set_name(src, "src"); - - auto ne_dst = ne; - for (int i = 0; i < dim; ++i) { - ne_dst[i] *= 2; - } - ggml_tensor * dst = ggml_new_tensor(ctx, type_dst, 4, ne_dst.data()); - ggml_set_param(dst); - ggml_set_name(dst, "dst"); - - size_t offset = 0; - for (int i = 0; i < dim; ++i) { - offset += ((ne_dst[i] - ne[i])/2)*dst->nb[i]; - } - ggml_tensor * out; - if (inplace) { - out = ggml_set_inplace(ctx, dst, src, - // The backward pass requires setting a contiguous region: - src->nb[1], src->nb[2], src->nb[3], offset); - } else { - out = ggml_set(ctx, dst, src, - // The backward pass requires setting a contiguous region: - src->nb[1], src->nb[2], src->nb[3], offset); - } - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_CPY -struct test_cpy : public test_case { - const ggml_type type_src; - const ggml_type type_dst; - const std::array ne_src; - const std::array ne_dst; - const std::array permute_src; - const std::array permute_dst; - const std::array dst_alloc; // if set, dst is a view into a larger buffer (strided) - bool _src_use_permute; - bool _dst_use_permute; - bool _src_transpose; - bool _use_dst_shape; - bool _use_dst_alloc; - - std::string vars() override { - if (_use_dst_alloc) { - return VARS_TO_STR8(type_src, type_dst, ne_src, ne_dst, permute_src, permute_dst, _src_transpose, dst_alloc); - } - if (_use_dst_shape) { - return VARS_TO_STR7(type_src, type_dst, ne_src, ne_dst, permute_src, permute_dst, _src_transpose); - } - return VARS_TO_STR6(type_src, type_dst, ne_src, permute_src, permute_dst, _src_transpose); - } - - int64_t total_elements() const { - return ne_src[0] * ne_src[1] * ne_src[2] * ne_src[3]; - } - - double max_nmse_err() override { - if (type_src == type_dst) { - return 0.0; - } - if (type_dst == GGML_TYPE_Q4_0 || type_dst == GGML_TYPE_Q4_1 || type_dst == GGML_TYPE_IQ4_NL || - type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1 || type_dst == GGML_TYPE_Q8_0) { - // estimate what the max nmse error would be if one quantized value is - // off by one. The test values are distributed in [-150,150], so it'll be - // roughly (150*2.0 / 2^bits)^2, divided by the mean square value of the reference, - // which is roughly 0.25*150^2 times the number of elements. - double err_estimate = 1.0f/8.0f * 150.0f; - if (type_dst == GGML_TYPE_IQ4_NL) { - // iq4_nl values are a bit more spread out - err_estimate *= 2.0f; - } - if (type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1) { - err_estimate /= 2.0f; - } - if (type_dst == GGML_TYPE_Q8_0) { - err_estimate /= 8.0f; - } - err_estimate *= err_estimate; - err_estimate /= (150.0f*150.0f*0.25f)*float(total_elements()); - return err_estimate; - } - return 1e-6; - } - - size_t op_size(ggml_tensor * t) override { - return ggml_nbytes(t) + ggml_nbytes(t->src[0]); - } - - test_cpy(ggml_type type_src = GGML_TYPE_F32, ggml_type type_dst = GGML_TYPE_F32, - std::array ne_src = {10, 10, 10, 1}, - std::array ne_dst = {-1, -1, -1, -1}, - std::array permute_src = {0, 0, 0, 0}, - std::array permute_dst = {0, 0, 0, 0}, - bool transpose_src = false, - std::array dst_alloc = {0, 0, 0, 0}) - : type_src(type_src), type_dst(type_dst), ne_src(ne_src), ne_dst(ne_dst), permute_src(permute_src), permute_dst(permute_dst), - dst_alloc(dst_alloc), - _src_use_permute(permute_src[0] + permute_src[1] + permute_src[2] + permute_src[3] > 0), - _dst_use_permute(permute_dst[0] + permute_dst[1] + permute_dst[2] + permute_dst[3] > 0), - _src_transpose(transpose_src), - _use_dst_shape(ne_dst[0] >= 0 && ne_dst[1] >= 0 && ne_dst[2] >= 0 && ne_dst[3] >= 0), - _use_dst_alloc(dst_alloc[0] > 0){} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * src = ggml_new_tensor(ctx, type_src, 4, ne_src.data()); - ggml_set_param(src); - ggml_set_name(src, "src"); - - if (_src_use_permute) { - src = ggml_permute(ctx, src, permute_src[0], permute_src[1], permute_src[2], permute_src[3]); - ggml_set_name(src, "src_permuted"); - } - - if (_src_transpose) { - src = ggml_transpose(ctx, src); - ggml_set_name(src, "src_transposed"); - } - - std::array dst_ne = _use_dst_shape ? ne_dst : std::array{src->ne[0], src->ne[1], src->ne[2], src->ne[3]}; - ggml_tensor * dst; - - if (_use_dst_alloc) { - // view a sub-block of a larger buffer -> strided dst - ggml_tensor * dst_buf = ggml_new_tensor(ctx, type_dst, 4, dst_alloc.data()); - ggml_set_name(dst_buf, "dst_buf"); - dst = ggml_view_4d(ctx, dst_buf, dst_ne[0], dst_ne[1], dst_ne[2], dst_ne[3], - dst_buf->nb[1], dst_buf->nb[2], dst_buf->nb[3], 0); - ggml_set_name(dst, "dst_view"); - } else { - dst = ggml_new_tensor(ctx, type_dst, 4, dst_ne.data()); - ggml_set_name(dst, "dst"); - - if (_dst_use_permute) { - dst = ggml_permute(ctx, dst, permute_dst[0], permute_dst[1], permute_dst[2], permute_dst[3]); - ggml_set_name(dst, "dst_permuted"); - } - } - - ggml_tensor * out = ggml_cpy(ctx, src, dst); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - // test extended range of values to check if casting between f32 and i32 is consistent - init_tensor_uniform(t, -150.f, 150.f); - } - } -}; - -// GGML_OP_CONT -struct test_cont : public test_case { - const ggml_type type; - const std::array ne; - bool use_view_slice; - - std::string vars() override { - return VARS_TO_STR3(type, ne, use_view_slice); - } - - test_cont(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 10, 10, 1}, - bool use_view_slice = false) - : type(type), ne(ne), use_view_slice(use_view_slice) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * src = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(src); - ggml_set_name(src, "src"); - - - ggml_tensor * dst; - if (use_view_slice) { - dst = ggml_view_4d(ctx, src, src->ne[0], 1, src->ne[2], src->ne[3], - src->nb[1], src->nb[2], src->nb[3], src->nb[0] * (src->ne[1] - 1)); - ggml_set_name(dst, "src_view_slice"); - } else { - dst = ggml_transpose(ctx, src); - ggml_set_name(dst, "src_transposed"); - } - - ggml_tensor * out = ggml_cont(ctx, dst); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_ADD -// GGML_OP_SUB -// GGML_OP_MUL -// GGML_OP_DIV -struct test_bin_bcast : public test_case { - using op_t = ggml_tensor * (*) (ggml_context *, ggml_tensor *, ggml_tensor *); - op_t op; - const ggml_type type; - const std::array ne; - const std::array nr; - int nf; // number of fused ops, nf == 1 -> single op (no fusion) - bool perm1; // permute src1? - bool src_overlap; // src0 and src1 are overlapping views of the same buffer - - bool run_whole_graph() override { return nf > 1; } - - std::string vars() override { - return VARS_TO_STR6(type, ne, nr, nf, perm1, src_overlap); - } - - size_t op_size(ggml_tensor * t) override { - return ggml_nbytes(t) * 3; - } - - test_bin_bcast(op_t op, ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 10, 1, 1}, - std::array nr = {1, 2, 1, 1}, - int nf = 1, - bool perm1 = false, bool src_overlap = false) - : op(op), type(type), ne(ne), nr(nr), nf(nf), perm1(perm1), src_overlap(src_overlap) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - GGML_ASSERT(nf <= 16); - - ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0]*nr[0], ne[1]*nr[1], ne[2]*nr[2], ne[3]*nr[3]); - ggml_set_name(a, "a"); - - ggml_tensor * b[16]; - for (int i = 0; i < nf; ++i) { - if (perm1) { - const int p[4] = { 1, 2, 0, 3 }; // hardcoded for now - - b[i] = ggml_new_tensor_4d(ctx, type, ne[p[0]], ne[p[1]], ne[p[2]], ne[p[3]]); - b[i] = ggml_permute(ctx, b[i], p[0], p[1], p[2], p[3]); - } else if (src_overlap) { - b[i] = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], 2 * (ne[3] / 3), a->nb[1], a->nb[2], a->nb[3], (ne[3] / 3) * a->nb[3]); - } else { - b[i] = ggml_new_tensor(ctx, type, 4, ne.data()); - } - ggml_set_name(b[i], (std::string("b") + std::to_string(i)).c_str()); - } - - // The backward pass supports broadcasting only for GGML_ADD: - const bool grad_supported = op == ggml_add && ggml_are_same_shape(a, b[0]) && nf == 1 && !perm1; - if (grad_supported) { - ggml_set_param(a); - ggml_set_param(b[0]); - } - - ggml_tensor *out; - - if (src_overlap) { - out = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], 2 * (ne[3] / 3), a->nb[1], a->nb[2], a->nb[3], 0); - } else { - out = a; - } - - for (int i = 0; i < nf; ++i) { - out = op(ctx, out, b[i]); - } - - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (op == ggml_mul || op == ggml_div) { - // MUL and DIV have numerical issues around zero: - init_tensor_uniform(t, 0.9f, 1.1f); - } else { - init_tensor_uniform(t); - } - } - } - - float grad_eps() override { - return 0.1f * (op == ggml_mul ? ne[0]*ne[1]*ne[2]*ne[3] : 1); - } - - bool grad_precise() override { - return op == ggml_div; - } - - double max_maa_err() override { - return op == ggml_add ? 1e-4 : 1e-3; - } -}; - -// GGML_OP_ADD_ID -struct test_add_id : public test_case { - const ggml_type type_a; - const ggml_type type_b; - const int64_t n_embd; - const int64_t n_experts; - const int64_t n_experts_used; - const int64_t n_token; - - std::string vars() override { - return VARS_TO_STR6(type_a, type_b, n_embd, n_experts, n_experts_used, n_token); - } - - size_t op_size(ggml_tensor * t) override { - return ggml_nbytes(t) + ggml_nbytes(t->src[0]) + ggml_nbytes(t->src[2]); - } - - test_add_id(ggml_type type_a = GGML_TYPE_F32, - ggml_type type_b = GGML_TYPE_F32, - int64_t n_embd = 128, - int64_t n_experts = 16, - int64_t n_experts_used = 8, - int64_t n_token = 10) - : type_a(type_a), type_b(type_b), n_embd(n_embd), - n_experts(n_experts), n_experts_used(n_experts_used), n_token(n_token) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_3d(ctx, type_a, n_embd, n_experts_used, n_token); - ggml_tensor * b = ggml_new_tensor_2d(ctx, type_b, n_embd, n_experts); - ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_experts, n_token); - if (n_experts_used != n_experts) { - ids = ggml_view_2d(ctx, ids, n_experts_used, n_token, ids->nb[1], 0); - ggml_set_name(ids, "view_of_ids"); - } - - ggml_tensor * out = ggml_add_id(ctx, a, b, ids); - ggml_set_name(out, "out"); - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I32) { - if (ggml_is_view_op(t->op)) { continue; } - std::random_device rd; - std::default_random_engine rng(rd()); - // ids - for (int64_t r = 0; r < ggml_nrows(t); r++) { - std::vector data(t->ne[0]); - for (int i = 0; i < t->ne[0]; i++) { - data[i] = i % n_experts; - } - std::shuffle(data.begin(), data.end(), rng); - ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); - } - } else { - init_tensor_uniform(t); - } - } - } -}; - -// GGML_OP_SCALE -struct test_scale : public test_case { - const ggml_type type; - const std::array ne; - float scale; - float bias; - bool inplace; - - std::string vars() override { - return VARS_TO_STR5(type, ne, scale, bias, inplace); - } - - test_scale(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 10, 10, 10}, - float scale = 2.0f, - float bias = 0.0f, - bool inplace = false) - : type(type), ne(ne), scale(scale), bias(bias), inplace(inplace) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out; - if (inplace) { - out = ggml_scale_bias_inplace(ctx, a, scale, bias); - } else { - out = ggml_scale_bias(ctx, a, scale, bias); - } - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_SCALE + GGML_UNARY_OP_TANH + GGML_OP_SCALE -struct test_softcap : public test_case { - const ggml_type type; - const std::array ne; - float softcap; - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "SOFTCAP"; - } - - bool run_whole_graph() override { return true; } - - std::string vars() override { - return VARS_TO_STR3(type, ne, softcap); - } - - test_softcap(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 10, 10, 10}, - float softcap = 30.0f) - : type(type), ne(ne), softcap(softcap) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_scale(ctx, ggml_tanh(ctx, ggml_scale(ctx, a, 1.0f / softcap)), softcap); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_SILU_BACK -struct test_silu_back : public test_case { - const ggml_type type; - const std::array ne; - float eps; - - std::string vars() override { - return VARS_TO_STR3(type, ne, eps); - } - - test_silu_back(ggml_type type = GGML_TYPE_F32, - std::array ne = {64, 5, 4, 3}, - float eps = 1e-6f) - : type(type), ne(ne), eps(eps) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - ggml_tensor * grad = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(grad, "grad"); - - ggml_tensor * out = ggml_silu_back(ctx, a, grad); - ggml_set_name(out, "out"); - - return out; - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_NORM -struct test_norm : public test_case { - const ggml_type type; - const std::array ne; - const bool v; // whether a is a non-contiguous view - const float eps; - const bool noncontig_rows; - - std::string vars() override { - return VARS_TO_STR5(type, ne, v, eps, noncontig_rows); - } - - test_norm(ggml_type type = GGML_TYPE_F32, - std::array ne = {64, 5, 4, 3}, - bool v = false, - float eps = 1e-6f, - bool noncontig_rows = false) - : type(type), ne(ne), v(v), eps(eps), noncontig_rows(noncontig_rows) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - const std::array ne_a = noncontig_rows ? - std::array{ ne[1], ne[0], ne[2], ne[3] } : ne; - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_set_name(a, "a"); - - if (noncontig_rows) { - a = ggml_permute(ctx, a, 1, 0, 2, 3); - ggml_set_name(a, "permuted a"); - } - if (v) { - a = ggml_view_4d(ctx, a, a->ne[0]/2, a->ne[1]/2, a->ne[2]/2, a->ne[3]/2, a->nb[1], a->nb[2], a->nb[3], 0); - ggml_set_name(a, "view of a"); - } - - ggml_tensor * out = ggml_norm(ctx, a, eps); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_NORM + GGML_OP_MUL + GGML_OP_ADD -struct test_norm_mul_add : public test_case { - const ggml_type type; - const std::array ne; - float eps; - const bool broadcast; - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "NORM_MUL_ADD"; - } - - bool run_whole_graph() override { return true; } - - std::string vars() override { - return VARS_TO_STR4(type, ne, eps, broadcast); - } - - test_norm_mul_add(ggml_type type = GGML_TYPE_F32, - std::array ne = {128, 2, 1, 1}, - float eps = 1e-5f, - bool broadcast = false) - : type(type), ne(ne), eps(eps), broadcast(broadcast) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - std::array broadcast_dims = {ne[0], ne[1] * 2, ne[2] * 2, ne[3] * 2}; - - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, broadcast ? broadcast_dims.data() : ne.data()); - ggml_tensor * w = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); ggml_set_param(w); ggml_set_param(b); - ggml_set_name(a, "a"); ggml_set_name(w, "w"); ggml_set_name(b, "b"); - - // Use a, w and b early to avoid OP_NONE in graph - a = ggml_add(ctx, ggml_add(ctx, a, w), b); - - ggml_tensor * n = ggml_norm(ctx, a, eps); - ggml_tensor * m = ggml_mul(ctx, n, w); - ggml_tensor * out = ggml_add(ctx, m, b); - ggml_set_name(out, "out"); - return out; - } -}; -// GGML_OP_RMS_NORM -struct test_rms_norm : public test_case { - const ggml_type type; - const std::array ne; - const bool v; // whether a is a non-contiguous view - const float eps; - const bool inplace; // whether to do the operation inplace - - std::string vars() override { - return VARS_TO_STR5(type, ne, v, eps, inplace); - } - - test_rms_norm(ggml_type type = GGML_TYPE_F32, - std::array ne = {64, 5, 4, 3}, - bool v = false, - float eps = 1e-6f, - bool inplace = false) - : type(type), ne(ne), v(v), eps(eps), inplace(inplace) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - if (v) { - a = ggml_view_4d(ctx, a, a->ne[0]/2, a->ne[1]/2, a->ne[2]/2, a->ne[3]/2, a->nb[1], a->nb[2], a->nb[3], 0); - ggml_set_name(a, "view of a"); - } - - ggml_tensor * out; - if (inplace) { - out = ggml_rms_norm_inplace(ctx, a, eps); - } else { - out = ggml_rms_norm(ctx, a, eps); - } - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -10.f, 10.f); - } - } - - float grad_eps() override { - return 1.0f; - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_RMS_NORM_BACK -struct test_rms_norm_back : public test_case { - const ggml_type type; - const std::array ne; - const float eps; - - std::string vars() override { - return VARS_TO_STR3(type, ne, eps); - } - - test_rms_norm_back(ggml_type type = GGML_TYPE_F32, - std::array ne = {64, 5, 4, 3}, - float eps = 1e-6f) - : type(type), ne(ne), eps(eps) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(b, "b"); - - ggml_tensor * out = ggml_rms_norm_back(ctx, a, b, eps); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -10.f, 10.f); - } - } -}; - -// GGML_OP_RMS_NORM + GGML_OP_MUL + GGML_OP_ADD -struct test_rms_norm_mul_add : public test_case { - const ggml_type type; - const std::array ne; - const float eps; - const bool broadcast; - const bool multi_add; // test a sequence of adds feeding into rms_norm - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "RMS_NORM_MUL_ADD"; - } - - bool run_whole_graph() override { return true; } - - std::string vars() override { - return VARS_TO_STR5(type, ne, eps, broadcast, multi_add); - } - - test_rms_norm_mul_add(ggml_type type = GGML_TYPE_F32, - std::array ne = {64, 5, 4, 3}, - float eps = 1e-6f, bool broadcast = false, bool multi_add = false) - : type(type), ne(ne), eps(eps), broadcast(broadcast), multi_add(multi_add) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - std::array broadcast_dims = {ne[0]*2, ne[1]*3, ne[2]*3, ne[3]*4}; - - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, broadcast ? broadcast_dims.data() : ne.data()); - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_tensor * c = ggml_new_tensor(ctx, type, 4, ne.data()); - - ggml_set_param(a); - ggml_set_name(a, "a"); - ggml_set_param(b); - ggml_set_name(b, "b"); - ggml_set_param(c); - ggml_set_name(c, "c"); - - // Use a, b and c early, so we don't end up with an OP_NONE between rms_norm and mul - a = ggml_add(ctx, ggml_add(ctx, a, b), c); - if (multi_add) { - a = ggml_add(ctx, ggml_add(ctx, a, b), c); - } - ggml_tensor * out = ggml_add(ctx, ggml_mul(ctx, ggml_rms_norm(ctx, a, eps), b), c); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -10.f, 10.f); - } - } - - float grad_eps() override { - return 1.0f; - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_ADD + GGML_OP_RMS_NORM (fused operation) -struct test_add_rms_norm : public test_case { - const ggml_type type; - const std::array ne; - const float eps; - const bool broadcast; - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "ADD_RMS_NORM"; - } - - bool run_whole_graph() override { return true; } - - std::string vars() override { - return VARS_TO_STR4(type, ne, eps, broadcast); - } - - test_add_rms_norm(ggml_type type = GGML_TYPE_F32, - std::array ne = {64, 5, 4, 3}, - float eps = 1e-6f, bool broadcast = false) - : type(type), ne(ne), eps(eps), broadcast(broadcast) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - std::array broadcast_dims = {ne[0]*2, ne[1]*3, ne[2]*3, ne[3]*4}; - - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, broadcast ? broadcast_dims.data() : ne.data()); - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); - - ggml_set_param(a); - ggml_set_name(a, "a"); - ggml_set_param(b); - ggml_set_name(b, "b"); - - // ADD operation followed by RMS_NORM - ggml_tensor * add_result = ggml_add(ctx, a, b); - ggml_set_name(add_result, "add_result"); - - ggml_tensor * out = ggml_rms_norm(ctx, add_result, eps); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -10.f, 10.f); - } - } - - float grad_eps() override { - return 1.0f; - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_UNARY(RELU) + GGML_OP_SQR (fused operation) -struct test_relu_sqr : public test_case { - const ggml_type type; - const std::array ne; - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "RELU_SQR"; - } - - bool run_whole_graph() override { return true; } - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_relu_sqr(ggml_type type = GGML_TYPE_F32, - std::array ne = {128, 2, 2, 2}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - ggml_tensor * r = ggml_relu(ctx, a); - ggml_set_name(r, "relu"); - - ggml_tensor * out = ggml_sqr(ctx, r); - ggml_set_name(out, "out"); - - return out; - } -}; - -// SNAKE activation fusion: y = x + sin(a*x)^2 * inv_b -// CUDA backend matches the naive 5-op chain (mul, sin, sqr, mul, add) -// and dispatches a single fused kernel. -struct test_snake_fuse : public test_case { - const ggml_type type; - const std::array ne; // [T, C, D2, D3] - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "SNAKE_FUSE"; - } - - bool run_whole_graph() override { return true; } - - double max_nmse_err() override { - // BF16 epsilon ~ 7.8e-3, F16 epsilon ~ 9.7e-4: relax tolerance to match - // the natural roundoff drift between the naive CPU chain and the fused - // CUDA kernel. F32 keeps the default tight bound. - switch (type) { - case GGML_TYPE_BF16: return 5e-3; - case GGML_TYPE_F16: return 5e-5; - default: return 1e-7; - } - } - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_snake_fuse(ggml_type type = GGML_TYPE_F32, - std::array ne = {256, 192, 1, 1}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * x = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); - ggml_set_name(x, "x"); - - ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, ne[1]); - ggml_set_name(a, "a"); - - ggml_tensor * inv_b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, ne[1]); - ggml_set_name(inv_b, "inv_b"); - - // exact 5-op chain that BigVGAN / Vocos frontends emit - ggml_tensor * ax = ggml_mul(ctx, x, a); - ggml_tensor * sin_ax = ggml_sin(ctx, ax); - ggml_tensor * sin_sq = ggml_sqr(ctx, sin_ax); - ggml_tensor * scaled = ggml_mul(ctx, sin_sq, inv_b); - ggml_tensor * out = ggml_add(ctx, x, scaled); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - // x in [-pi, pi] to exercise sin periodicity, params in default [-1, 1] - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { - const std::string name = ggml_get_name(t); - if (name == "x") { - init_tensor_uniform(t, -3.14159f, 3.14159f); - } else { - init_tensor_uniform(t); - } - } - } -}; - -// GGML_OP_SSM_CONV -struct test_ssm_conv : public test_case { - const ggml_type type; - const std::array ne_a; - const std::array ne_b; - - std::string vars() override { - return VARS_TO_STR3(type, ne_a, ne_b); - } - - test_ssm_conv(ggml_type type = GGML_TYPE_F32, - std::array ne_a = {10, 10, 10, 1}, - std::array ne_b = {3, 3, 1, 1}) - : type(type), ne_a(ne_a), ne_b(ne_b) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne_b.data()); - ggml_tensor * out = ggml_ssm_conv(ctx, a, b); - return out; - } -}; - -// GGML_OP_SSM_CONV + GGML_OP_ADD (channel-wise bias, optional) + GGML_OP_UNARY(SILU) (fused operation) -struct test_ssm_conv_bias_silu : public test_case { - const ggml_type type; - const std::array ne_a; - const std::array ne_b; - const bool fuse_bias; - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "SSM_CONV_BIAS_SILU"; - } - - bool run_whole_graph() override { return true; } - - std::string vars() override { - return VARS_TO_STR4(type, ne_a, ne_b, fuse_bias); - } - - test_ssm_conv_bias_silu(ggml_type type, std::array ne_a, std::array ne_b, - bool fuse_bias) - : type(type), ne_a(ne_a), ne_b(ne_b), fuse_bias(fuse_bias) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne_b.data()); - ggml_set_name(a, "a"); - ggml_set_name(b, "b"); - - ggml_tensor * out = ggml_ssm_conv(ctx, a, b); - - if (fuse_bias) { - ggml_tensor * bias = ggml_new_tensor_1d(ctx, type, out->ne[0]); - ggml_set_name(bias, "bias"); - out = ggml_add(ctx, out, bias); - } - - out = ggml_silu(ctx, out); - - ggml_set_name(out, "out"); - return out; - } -}; - -// GGML_OP_SSM_SCAN -struct test_ssm_scan : public test_case { - const ggml_type type; - - const int64_t d_state; - const int64_t head_dim; - const int64_t n_head; - const int64_t n_group; - const int64_t n_seq_tokens; - const int64_t n_seqs; - const bool xbc_overlap; - - std::string vars() override { - return VARS_TO_STR8(type, d_state, head_dim, n_head, n_group, n_seq_tokens, n_seqs, xbc_overlap); - } - - test_ssm_scan(ggml_type type = GGML_TYPE_F32, - int64_t d_state = 32, - int64_t head_dim = 1, // non-zero for Mamba-2 - int64_t n_head = 32, - int64_t n_group = 1, - int64_t n_seq_tokens = 32, - int64_t n_seqs = 32, - bool xbc_overlap = false) - : type(type), d_state(d_state), head_dim(head_dim), n_head(n_head), n_group(n_group), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs), xbc_overlap(xbc_overlap) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * s = ggml_new_tensor_4d(ctx, type, d_state, head_dim, n_head, n_seqs); - ggml_tensor * dt = ggml_new_tensor_3d(ctx, type, n_head, n_seq_tokens, n_seqs); - ggml_tensor * A = ggml_new_tensor_2d(ctx, type, (head_dim > 1) ? 1 : d_state, n_head); - ggml_tensor * x; - ggml_tensor * B; - ggml_tensor * C; - - if (xbc_overlap) { - ggml_tensor * xbc = ggml_new_tensor_4d(ctx, type, d_state, n_head, n_seq_tokens, 2 * n_seqs); - x = ggml_view_4d(ctx, xbc, head_dim, n_head, n_seq_tokens, n_seqs, - xbc->nb[1], xbc->nb[2], xbc->nb[3], xbc->nb[3]); - B = ggml_view_4d(ctx, xbc, d_state, n_group, n_seq_tokens, n_seqs, - xbc->nb[1], xbc->nb[2], xbc->nb[3], 0); - C = ggml_view_4d(ctx, xbc, d_state, n_group, n_seq_tokens, n_seqs, - xbc->nb[1], xbc->nb[2], xbc->nb[3], 2 * xbc->nb[3]); - } else { - x = ggml_new_tensor_4d(ctx, type, head_dim, n_head, n_seq_tokens, n_seqs); - B = ggml_new_tensor_4d(ctx, type, d_state, n_group, n_seq_tokens, n_seqs); - C = ggml_new_tensor_4d(ctx, type, d_state, n_group, n_seq_tokens, n_seqs); - } - ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_seqs); - ggml_tensor * out = ggml_ssm_scan(ctx, s, x, dt, A, B, C, ids); - return out; - } - - // similar to test_mul_mat_id - void initialize_tensors(ggml_context * ctx) override { - std::random_device rd; - std::default_random_engine rng(rd()); - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I32) { - if (ggml_is_view_op(t->op)) { continue; } - // ids - for (int64_t r = 0; r < ggml_nrows(t); r++) { - std::vector data(t->ne[0]); - for (int i = 0; i < t->ne[0]; i++) { - data[i] = i; - } - std::shuffle(data.begin(), data.end(), rng); - ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); - } - } else { - init_tensor_uniform(t); - } - } - } -}; - -// GGML_OP_RWKV_WKV6 -struct test_rwkv_wkv6 : public test_case { - const ggml_type type; - - const int64_t head_count; - const int64_t head_size; - const int64_t n_seq_tokens; - const int64_t n_seqs; - - std::string vars() override { - return VARS_TO_STR5(type, head_count, head_size, n_seq_tokens, n_seqs); - } - - test_rwkv_wkv6(ggml_type type = GGML_TYPE_F32, - int64_t head_count = 32, int64_t head_size = 64, int64_t n_seq_tokens = 32, int64_t n_seqs = 32) - : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - const int64_t n_tokens = n_seq_tokens * n_seqs; - ggml_tensor * r = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - ggml_tensor * k = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - ggml_tensor * v = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - ggml_tensor * tf = ggml_new_tensor(ctx, type, 2, std::vector{ head_size, head_count }.data()); - ggml_tensor * td = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - ggml_tensor * s = ggml_new_tensor(ctx, type, 2, std::vector{ head_size * head_size * head_count, n_seqs }.data()); - ggml_tensor * out = ggml_rwkv_wkv6(ctx, k, v, r, tf, td, s); - return out; - } -}; - -// GGML_OP_GATED_DELTA_NET -struct test_gated_delta_net : public test_case { - const ggml_type type; - - const int64_t head_count; - const int64_t head_size; - const int64_t n_seq_tokens; - const int64_t n_seqs; - const int v_repeat; - const bool permuted; - const bool kda; - const int64_t K; // snapshot slot count: 1 = final-only, >1 = last K states - - std::string vars() override { - return VARS_TO_STR9(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K); - } - - test_gated_delta_net(ggml_type type = GGML_TYPE_F32, - int64_t head_count = 4, int64_t head_size = 16, int64_t n_seq_tokens = 1, int64_t n_seqs = 1, - int v_repeat = 1, bool permuted = false, bool kda = false, int64_t K = 1) - : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs), - v_repeat(v_repeat), permuted(permuted), kda(kda), K(K) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * q; - ggml_tensor * k; - ggml_tensor * v; - if (permuted) { - // create with dims 1 and 2 swapped, then permute back to get non-contiguous layout - q = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs), 0, 2, 1, 3); - k = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs), 0, 2, 1, 3); - v = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count * v_repeat, n_seqs), 0, 2, 1, 3); - } else { - q = ggml_new_tensor_4d(ctx, type, head_size, head_count, n_seq_tokens, n_seqs); - k = ggml_new_tensor_4d(ctx, type, head_size, head_count, n_seq_tokens, n_seqs); - v = ggml_new_tensor_4d(ctx, type, head_size, head_count * v_repeat, n_seq_tokens, n_seqs); - } - ggml_set_name(q, "q"); - ggml_set_name(k, "k"); - ggml_set_name(v, "v"); - const int64_t g_ne0 = kda ? head_size : 1; - ggml_tensor * g = ggml_new_tensor_4d(ctx, type, g_ne0, head_count * v_repeat, n_seq_tokens, n_seqs); - ggml_tensor * beta = ggml_new_tensor_4d(ctx, type, 1, head_count * v_repeat, n_seq_tokens, n_seqs); - ggml_tensor * state = ggml_new_tensor_4d(ctx, type, head_size, head_size, head_count * v_repeat, n_seqs); - ggml_set_name(g, "g"); - ggml_set_name(beta, "beta"); - ggml_set_name(state, "state"); - // q/k are L2-normalised in qwen35/kimi-linear before delta_net - q = ggml_l2_norm(ctx, q, 1e-6f); - k = ggml_l2_norm(ctx, k, 1e-6f); - ggml_tensor * out = ggml_gated_delta_net(ctx, q, k, v, g, beta, state, K); - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { - if (ggml_is_view_op(t->op)) { continue; } - if (strcmp(t->name, "g") == 0) { - init_tensor_uniform(t, -20.0f, -1e-4f); - } else if (strcmp(t->name, "beta") == 0) { - init_tensor_uniform(t, 0.0f, 1.0f); - } else if (strcmp(t->name, "v") == 0) { - init_tensor_uniform(t, -0.3f, 5.0f); - } else { - init_tensor_uniform(t); - } - } - } -}; - -// GGML_OP_GATED_LINEAR_ATTN -struct test_gla : public test_case { - const ggml_type type; - - const int64_t head_count; - const int64_t head_size; - const int64_t n_seq_tokens; - const int64_t n_seqs; - - std::string vars() override { - return VARS_TO_STR5(type, head_count, head_size, n_seq_tokens, n_seqs); - } - - test_gla(ggml_type type = GGML_TYPE_F32, - int64_t head_count = 32, int64_t head_size = 64, int64_t n_seq_tokens = 32, int64_t n_seqs = 32) - : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - const int64_t n_tokens = n_seq_tokens * n_seqs; - ggml_tensor * q = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - ggml_tensor * k = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - ggml_tensor * v = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - ggml_tensor * g = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - ggml_tensor * s = ggml_new_tensor(ctx, type, 2, std::vector{ head_size * head_size * head_count, n_seqs }.data()); - ggml_tensor * out = ggml_gated_linear_attn(ctx, k, v, q, g, s, pow(head_size, -0.5)); - return out; - } -}; - -// GGML_OP_RWKV_WKV7 -struct test_rwkv_wkv7 : public test_case { - const ggml_type type; - - const int64_t head_count; - const int64_t head_size; - const int64_t n_seq_tokens; - const int64_t n_seqs; - - std::string vars() override { - return VARS_TO_STR5(type, head_count, head_size, n_seq_tokens, n_seqs); - } - - test_rwkv_wkv7(ggml_type type = GGML_TYPE_F32, - int64_t head_count = 32, int64_t head_size = 64, int64_t n_seq_tokens = 32, int64_t n_seqs = 32) - : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - const int64_t n_tokens = n_seq_tokens * n_seqs; - ggml_tensor * r = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - ggml_tensor * w = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - ggml_tensor * k = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - ggml_tensor * v = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - ggml_tensor * a = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - ggml_tensor * b = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); - // Outputs may become NaN with long seqlen without these normalization - a = ggml_l2_norm(ctx, a, 1e-7F); - b = ggml_l2_norm(ctx, b, 1e-7F); - ggml_tensor * s = ggml_new_tensor(ctx, type, 2, std::vector{ head_size * head_size * head_count, n_seqs }.data()); - ggml_tensor * out = ggml_rwkv_wkv7(ctx, r, w, k, v, a, b, s); - return out; - } -}; - -// GGML_OP_MUL_MAT -struct test_mul_mat : public test_case { - const ggml_type type_a; - const ggml_type type_b; - const int64_t m; - const int64_t n; - const int64_t k; - const std::array bs; // dims 3 and 4 - const std::array nr; // repeat in dims 3 and 4 - const std::array per; // permutation of dimensions - const int64_t k_v; // size of k in memory, resulting in a non-contiguous view for k_v > k, no view for k_v == 0 - const uint32_t o; // number of outputs - - std::string vars() override { - return VARS_TO_STR10(type_a, type_b, m, n, k, bs, nr, per, k_v, o); - } - - double max_nmse_err() override { - return 5e-4; - } - - double max_nmse_err(ggml_backend_t backend) override { - // for blackwell we quantize activations to mxfp4 instead of q8_1 so we add higher tolerance - if ((type_a == GGML_TYPE_MXFP4 || type_a == GGML_TYPE_NVFP4) && backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) { - return 2e-2; - } - return max_nmse_err(); - } - - int64_t grad_nmax() override { - return 20000; - } - - uint64_t op_flops(ggml_tensor * t) override { - GGML_UNUSED(t); - return 2 * m * n * k * bs[0] * nr[0] * bs[1] * nr[1]; - } - - test_mul_mat(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, - int64_t m = 32, int64_t n = 32, int64_t k = 32, - std::array bs = {10, 10}, - std::array nr = {2, 2}, - std::array per = {0, 1, 2, 3}, - int64_t k_v = 0, uint32_t o = 1) - : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), per(per), k_v(k_v), o(o) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - // C^T = A * B^T: (k, m) * (k, n) => (m, n) - ggml_tensor * a; - ggml_tensor * b; - - const int npermuted = (per[0] != 0) + (per[1] != 1) + (per[2] != 2) + (per[3] != 3); - if (npermuted > 0) { - GGML_ASSERT(npermuted == 2); - GGML_ASSERT(k_v == 0); // not handled - GGML_ASSERT(!ggml_is_quantized(type_a) || per[0] == 0); - GGML_ASSERT(!ggml_is_quantized(type_b) || per[0] == 0); - - // Create tensors with the permuted dimensions, then permute them back to the dimensions given by m,n,k. - const int64_t ne_a[4] = {k, m, bs[0], bs[1]}; - const int64_t ne_b[4] = {k, n, bs[0]*nr[0], bs[1]*nr[1]}; - - a = ggml_new_tensor_4d(ctx, type_a, ne_a[per[0]], ne_a[per[1]], ne_a[per[2]], ne_a[per[3]]); - b = ggml_new_tensor_4d(ctx, type_b, ne_b[per[0]], ne_b[per[1]], ne_b[per[2]], ne_b[per[3]]); - if (!ggml_is_quantized(type_a)) { - if (bs[1] == 1 && nr[1] == 1) { - ggml_set_param(a); - } - ggml_set_param(b); - } - ggml_set_name(a, "a"); - ggml_set_name(b, "b"); - - a = ggml_permute(ctx, a, per[0], per[1], per[2], per[3]); - b = ggml_permute(ctx, b, per[0], per[1], per[2], per[3]); - ggml_set_name(a, "a_permuted"); - ggml_set_name(b, "b_permuted"); - } else { - const int64_t k_physical = k_v == 0 ? k : k_v; - a = ggml_new_tensor_4d(ctx, type_a, k_physical, m, bs[0], bs[1]); - b = ggml_new_tensor_4d(ctx, type_b, k_physical, n, bs[0]*nr[0], bs[1]*nr[1]); - - if (!ggml_is_quantized(type_a)) { - if (bs[1] == 1 && nr[1] == 1) { - ggml_set_param(a); - } - ggml_set_param(b); - } - - if (k_v != 0) { - GGML_ASSERT(k_v > k); - a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], a->nb[2], a->nb[3], 0); - b = ggml_view_4d(ctx, b, k, n, bs[0]*nr[0], bs[1]*nr[1], b->nb[1], b->nb[2], b->nb[3], 0); - } - ggml_set_name(a, "a"); - ggml_set_name(b, "b"); - } - - ggml_tensor * out = ggml_mul_mat(ctx, a, b); - ggml_set_name(out, "out"); - for (uint32_t i = 1; i < o; ++i) { - ggml_tensor * out2 = ggml_mul_mat(ctx, a, b); - ggml_set_name(out2, "out2"); - out = ggml_add(ctx, out, out2); - } - - return out; - } - - bool run_whole_graph() override { return o > 1; } - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return ggml_op_name(GGML_OP_MUL_MAT); - } -}; - -// GGML_HINT_SRC0_IS_HADAMARD -struct test_mul_mat_hadamard : public test_mul_mat { - test_mul_mat_hadamard(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, - int64_t m = 32, int64_t n = 32, int64_t k = 32, - std::array bs = {1, 1}, - std::array nr = {1, 1}) - : test_mul_mat(type_a, type_b, m, n, k, bs, nr) { - GGML_ASSERT(type_a == GGML_TYPE_F32); - } - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * out = test_mul_mat::build_graph(ctx); - // Find the mul_mat op in the graph and set the hint - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->op == GGML_OP_MUL_MAT) { - ggml_mul_mat_set_hint(t, GGML_HINT_SRC0_IS_HADAMARD); - } - } - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (strcmp(t->name, "a") == 0) { - const int64_t n_cols = t->ne[0]; - const int64_t n_rows = ggml_nrows(t); - std::vector data(n_cols * n_rows); - float scale = 1.0f / sqrtf((float)n_cols); - for (int64_t r = 0; r < n_rows; r++) { - float * row_data = data.data() + r * n_cols; - for (int64_t i = 0; i < n_cols; i++) { - int pop = 0; - int64_t val = r & i; - while (val) { - pop += (val & 1); - val >>= 1; - } - row_data[i] = (pop % 2 == 0) ? scale : -scale; - } - } - ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(float)); - } else if (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_F16) { - init_tensor_uniform(t); - } - } - } - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "MUL_MAT_HADAMARD"; - } -}; - -static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats) { - std::random_device rd; - std::default_random_engine rng(rd()); - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I32) { - if (ggml_is_view_op(t->op)) { continue; } - // ids - for (int64_t r = 0; r < ggml_nrows(t); r++) { - std::vector data(t->ne[0]); - for (int i = 0; i < t->ne[0]; i++) { - data[i] = i % n_mats; - } - std::shuffle(data.begin(), data.end(), rng); - ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); - } - } else { - init_tensor_uniform(t); - } - } -} - -// GGML_OP_MUL_MAT_ID -struct test_mul_mat_id : public test_case { - const ggml_type type_a; - const ggml_type type_b; - const int n_mats; - const int n_used; - const bool b; // broadcast b matrix - const int64_t m; - const int64_t n; - const int64_t k; - - std::string vars() override { - return VARS_TO_STR8(type_a, type_b, n_mats, n_used, b, m, n, k); - } - - double max_nmse_err() override { - return 5e-4; - } - - double max_nmse_err(ggml_backend_t backend) override { - // for blackwell we quantize activations to mxfp4 instead of q8_1 so we add higher tolerance - if ((type_a == GGML_TYPE_MXFP4 || type_a == GGML_TYPE_NVFP4) && backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) { - return 2e-2; - } - return max_nmse_err(); - } - - uint64_t op_flops(ggml_tensor * t) override { - GGML_UNUSED(t); - return 2 * m * k * n * n_used; - } - - test_mul_mat_id(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, - int n_mats = 8, int n_used = 2, bool b = false, - int64_t m = 32, int64_t n = 32, int64_t k = 32) - : type_a(type_a), type_b(type_b), n_mats(n_mats), n_used(n_used), b(b), - m(m), n(n), k(k) { - GGML_ASSERT(n_used <= n_mats); - } - - ggml_tensor * build_graph(ggml_context * ctx) override { - // C^T = A * B^T: (k, m) * (k, n) => (m, n) - ggml_tensor * as = ggml_new_tensor_3d(ctx, type_a, k, m, n_mats); - ggml_set_name(as, "as"); - - ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n); - ggml_set_name(ids, "ids"); - if (n_used != n_mats) { - ids = ggml_view_2d(ctx, ids, n_used, n, ids->nb[1], 0); - ggml_set_name(ids, "view_of_ids"); - } - - ggml_tensor * b = ggml_new_tensor_3d(ctx, type_b, k, this->b ? 1 : n_used, n); - ggml_set_name(b, "b"); - - ggml_tensor * out = ggml_mul_mat_id(ctx, as, b, ids); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - init_mul_mat_id_tensors(ctx, n_mats); - } -}; - -// GGML_OP_MUL_MAT_ID + GGML_OP_ADD or GGML_OP_MUL -struct test_mul_mat_id_fusion : public test_case { - const ggml_type type_a; - const ggml_type type_b; - const int n_mats; - const int n_used; - const bool b; // broadcast b matrix - const int64_t m; - const int64_t n; - const int64_t k; - const uint32_t o; // number of outputs - const bool mul; - - std::string vars() override { - return VARS_TO_STR10(type_a, type_b, n_mats, n_used, b, m, n, k, o, mul); - } - - double max_nmse_err() override { - return 5e-4; - } - - uint64_t op_flops(ggml_tensor * t) override { - GGML_UNUSED(t); - return 2 * m * k * n * n_used; - } - - test_mul_mat_id_fusion(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, - int n_mats = 8, int n_used = 2, bool b = false, - int64_t m = 32, int64_t n = 32, int64_t k = 32, uint32_t o = 1, bool mul = false) - : type_a(type_a), type_b(type_b), n_mats(n_mats), n_used(n_used), b(b), - m(m), n(n), k(k), o(o), mul(mul) { - GGML_ASSERT(n_used <= n_mats); - } - - ggml_tensor * build_graph(ggml_context * ctx) override { - // C^T = A * B^T: (k, m) * (k, n) => (m, n) - ggml_tensor * as = ggml_new_tensor_3d(ctx, type_a, k, m, n_mats); - ggml_set_name(as, "as"); - - ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n); - ggml_set_name(ids, "ids"); - if (n_used != n_mats) { - ids = ggml_view_2d(ctx, ids, n_used, n, ids->nb[1], 0); - ggml_set_name(ids, "view_of_ids"); - } - - ggml_tensor * b = ggml_new_tensor_3d(ctx, type_b, k, this->b ? 1 : n_used, n); - ggml_set_name(b, "b"); - - ggml_tensor * out = ggml_mul_mat_id(ctx, as, b, ids); - ggml_set_name(out, "out"); - - for (uint32_t i = 1; i < o; ++i) { - ggml_tensor * a2 = ggml_new_tensor_3d(ctx, type_a, k, m, n_mats); - ggml_tensor * out2 = ggml_mul_mat_id(ctx, a2, b, ids); - ggml_set_name(out2, "out2"); - out = ggml_add(ctx, out, out2); - } - - if (mul) { - std::array ne { 1, out->ne[1], out->ne[2], out->ne[3] }; - ne[0] = 1; - ggml_tensor * m = ggml_new_tensor(ctx, out->type, 4, ne.data()); - out = ggml_mul(ctx, out, m); - } - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - init_mul_mat_id_tensors(ctx, n_mats); - } - - bool run_whole_graph() override { return true; } - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "MUL_MAT_ID_FUSION"; - } -}; - -// GGML_OP_OUT_PROD -struct test_out_prod : public test_case { - const ggml_type type_a; - const ggml_type type_b; - const int64_t m; - const int64_t n; - const int64_t k; - const std::array bs; // dims 3 and 4 - const std::array nr; // repeat in dims 3 and 4 - const bool trans_b; - - std::string vars() override { - return VARS_TO_STR8(type_a, type_b, m, n, k, bs, nr, trans_b); - } - - double max_nmse_err() override { - return 5e-4; - } - - test_out_prod(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, - int64_t m = 32, int64_t n = 32, int64_t k = 32, - std::array bs = {10, 10}, - std::array nr = {2, 2}, - bool trans_b = false) - : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), trans_b(trans_b) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_4d(ctx, type_a, m, k, bs[0], bs[1]); - ggml_set_name(a, "a"); - - ggml_tensor * b; - if (trans_b) { - b = ggml_new_tensor_4d(ctx, type_b, k, n, bs[0]*nr[0], bs[1]*nr[1]); - b = ggml_transpose(ctx, b); - } else { - b = ggml_new_tensor_4d(ctx, type_b, n, k, bs[0]*nr[0], bs[1]*nr[1]); - } - ggml_set_name(b, "b"); - - ggml_tensor * out = ggml_out_prod(ctx, a, b); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_SQR -struct test_sqr : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_sqr(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_sqr(ctx, a); - ggml_set_name(out, "out"); - - return out; - } - - float grad_eps() override { - return 0.1f * 0.25f*ne[0]*ne[1]*ne[2]*ne[3]; // 10% of expected value of sum. - } -}; - -// GGML_OP_SQRT -struct test_sqrt : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_sqrt(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 3, 3, 2}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_sqrt(ctx, a); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - // fill with positive values - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, 50.0f, 100.0f); - } - } - - float grad_eps() override { - return 20.0f; - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_LOG -struct test_log : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_log(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_log(ctx, a); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - // log(1) == 0, cluster values there to keep the sum low for better precision in the backward pass: - init_tensor_uniform(t, 0.9f, 1.1f); - } - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_SIN -struct test_sin : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_sin(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 2, 2, 2}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_sin(ctx, a); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -6.5f, 6.5f); // Covers interval [-2*pi, 2*pi]. - } - } - - double max_maa_err() override { - return 1e-3; - } - - float grad_eps() override { - return 0.2f; - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_COS -struct test_cos : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_cos(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 2, 2, 2}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_cos(ctx, a); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -6.5f, 6.5f); // Covers interval [-2*pi, 2*pi]. - } - } - - double max_maa_err() override { - return 1e-3; - } - - float grad_eps() override { - return 0.2f; - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_CLAMP -struct test_clamp : public test_case { - const ggml_type type; - const std::array ne; - float min; - float max; - - std::string vars() override { - return VARS_TO_STR4(type, ne, min, max); - } - - test_clamp(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}, - float min = -0.5f, float max = 0.5f) - : type(type), ne(ne), min(min), max(max) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_clamp(ctx, a, min, max); - ggml_set_name(out, "out"); - - return out; - } - - float grad_eps() override { - return 1e-2f; - } - - std::vector grad_expect() override { - return {0.0f, 1.0f}; - } -}; - -// GGML_OP_FLOOR -struct test_floor : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_floor(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 2, 2, 2}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_floor(ctx, a); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -10.0f, 10.0f); - } - } -}; - -// GGML_OP_CEIL -struct test_ceil : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_ceil(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 2, 2, 2}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_ceil(ctx, a); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -10.0f, 10.0f); - } - } -}; - -// GGML_OP_ROUND -struct test_round : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_round(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 2, 2, 2}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_round(ctx, a); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -10.0f, 10.0f); - } - } -}; - -// GGML_OP_TRUNC -struct test_trunc : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_trunc(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 2, 2, 2}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_trunc(ctx, a); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -10.0f, 10.0f); - } - } -}; - -// GGML_OP_DIAG_MASK_INF -struct test_diag_mask_inf : public test_case { - const ggml_type type; - const std::array ne; - const int n_past; - - std::string vars() override { - return VARS_TO_STR3(type, ne, n_past); - } - - test_diag_mask_inf(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 10, 3, 2}, - int n_past = 5) - : type(type), ne(ne), n_past(n_past) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_diag_mask_inf(ctx, a, n_past); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_SOFT_MAX -struct test_soft_max : public test_case { - const ggml_type type; - const std::array ne; - const bool mask; - const bool sinks; - const ggml_type m_prec; - const std::array nr23; // broadcast only dims 2 and 3 - const float scale; - const float max_bias; - const bool inplace; - - std::string vars() override { - return VARS_TO_STR9(type, ne, mask, sinks, m_prec, nr23, scale, max_bias, inplace); - } - - // the 1024 test with bias occasionally fails: - // SOFT_MAX(type=f32,ne=[1024,16,1,1],mask=1,scale=1.000000,max_bias=8.000000): [SOFT_MAX] NMSE = 0.000000103 > 0.000000100 FAIL - virtual double max_nmse_err() override { - return 1e-6; - } - - test_soft_max(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}, - bool mask = false, - bool sinks = false, - ggml_type m_prec = GGML_TYPE_F32, - std::array nr23 = {1, 1}, - float scale = 1.0f, - float max_bias = 0.0f, - bool inplace = false) - : type(type), ne(ne), mask(mask), sinks(sinks), m_prec(m_prec), nr23(nr23), scale(scale), max_bias(max_bias), inplace(inplace) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2]*nr23[0], ne[3]*nr23[1]); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * mask = nullptr; - if (this->mask) { - mask = ggml_new_tensor_4d(ctx, m_prec, ne[0], ne[1], ne[2], ne[3]); - ggml_set_name(mask, "mask"); - } - - ggml_tensor * sinks = nullptr; - if (this->sinks) { - sinks = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[2]*nr23[0]); - ggml_set_name(sinks, "sinks"); - } - - ggml_tensor * out; - if (inplace) { - out = ggml_soft_max_ext_inplace(ctx, a, mask, scale, max_bias); - } else { - out = ggml_soft_max_ext(ctx, a, mask, scale, max_bias); - } - ggml_soft_max_add_sinks(out, sinks); - ggml_set_name(out, "out"); - - return out; - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_SOFT_MAX_BACK -struct test_soft_max_back : public test_case { - const ggml_type type; - const std::array ne; - const float scale; - const float max_bias; - - std::string vars() override { - return VARS_TO_STR4(type, ne, scale, max_bias); - } - - test_soft_max_back(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}, - float scale = 1.0f, - float max_bias = 0.0f) - : type(type), ne(ne), scale(scale), max_bias(max_bias) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_soft_max_ext_back(ctx, a, b, scale, max_bias); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_ROPE + GGML_OP_ROPE_BACK -struct test_rope : public test_case { - const ggml_type type; - const std::array ne_a; - int n_dims; - int mode; - int n_ctx; // used to generate positions - float fs; // freq_scale - float ef; // ext_factor - float af; // attn_factor - bool ff; - int v; // view (1 : non-contiguous a) - bool forward; - bool inplace; - - std::string vars() override { - // forward can be inferred from the op, does not need to be printed - return VARS_TO_STR11(type, ne_a, n_dims, mode, n_ctx, fs, ef, af, ff, v, inplace); - } - - test_rope(ggml_type type = GGML_TYPE_F32, - std::array ne_a = {10, 5, 3, 1}, - int n_dims = 10, int mode = GGML_ROPE_TYPE_NORMAL, int n_ctx = 512, float fs = 1.0f, - float ef = 0.0f, float af = 0.0f, bool ff = false, int v = 0, bool forward = true, bool inplace = false) - : type(type), ne_a(ne_a), n_dims(n_dims), mode(mode), n_ctx(n_ctx), fs(fs), ef(ef), af(af), ff(ff), v(v), forward(forward), inplace(inplace) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a; - if (v & 1) { - auto ne = ne_a; ne[0] *= 2; ne[1] *= 4; ne[2] *= 3; - a = ggml_new_tensor(ctx, type, 4, ne.data()); - if (forward) { - ggml_set_param(a); - } - ggml_set_name(a, "a"); - - a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); - ggml_set_name(a, "view_of_a"); - } else if (v == 2) { - // second-half slice along dim 0 (mimics build_rope_2d in clip.cpp). - // The non-zero view offset (ne_a[0] * elem_size) often produces a - // non-aligned buffer offset, which exercises backends' alignment paths. - auto ne = ne_a; ne[0] *= 2; - a = ggml_new_tensor(ctx, type, 4, ne.data()); - if (forward) { - ggml_set_param(a); - } - ggml_set_name(a, "a"); - - a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], - a->nb[1], a->nb[2], a->nb[3], - ne_a[0] * ggml_element_size(a)); - ggml_set_name(a, "view_of_a"); - } else { - a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - if (forward) { - ggml_set_param(a); - } - ggml_set_name(a, "a"); - } - - const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; - const bool is_vision = mode == GGML_ROPE_TYPE_VISION; - - ggml_tensor * pos; - if (is_mrope || is_vision) { - pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne_a[2] * 4); - } else { - pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne_a[2]); - } - ggml_set_name(pos, "pos"); - - ggml_tensor * freq = nullptr; - if (ff) { - freq = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_dims/2); - ggml_set_name(freq, "freq"); - } - - ggml_tensor * out; - if (is_mrope) { - if (is_vision) { - GGML_ASSERT(n_dims/4 > 0); - int rope_sections[4] = {n_dims/4, n_dims/4, 0, 0}; // Vision-RoPE only use first two dimension for image (x, y) coordinate - if (forward) { - if (inplace) { - out = ggml_rope_multi_inplace(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); - } else { - out = ggml_rope_multi(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); - } - } else { - out = ggml_rope_multi_back(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); - } - } else { - GGML_ASSERT(n_dims/3 > 0); - int rope_sections[4] = {n_dims/3, n_dims/3, n_dims/3, 0}; - if (forward) { - if (inplace) { - out = ggml_rope_multi_inplace(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); - } else { - out = ggml_rope_multi(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); - } - } else { - out = ggml_rope_multi_back(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); - } - } - } else { - if (forward) { - if (inplace) { - out = ggml_rope_ext_inplace(ctx, a, pos, freq, n_dims, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); - } else { - out = ggml_rope_ext(ctx, a, pos, freq, n_dims, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); - } - } else { - out = ggml_rope_ext_back(ctx, a, pos, freq, n_dims, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); - } - } - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I32) { - // pos - const int num_pos_ids = (mode & GGML_ROPE_TYPE_MROPE) ? ne_a[2] * 4 : ne_a[2]; - std::vector data(num_pos_ids); - for (int i = 0; i < num_pos_ids; i++) { - data[i] = rand() % n_ctx; - } - ggml_backend_tensor_set(t, data.data(), 0, num_pos_ids * sizeof(int)); - } else { - if (t->ne[0] == n_dims/2) { - // frequency factors in the range [0.9f, 1.1f] - init_tensor_uniform(t, 0.9f, 1.1f); - } else { - init_tensor_uniform(t); - } - } - } - } - - double max_maa_err() override { - return 1e-3; - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_POOL2D -struct test_pool2d : public test_case { - enum ggml_op_pool pool_type; - const ggml_type type_input; - const std::array ne_input; - // kernel size - const int k0; - const int k1; - // stride - const int s0; - const int s1; - // padding - const int p0; - const int p1; - - std::string vars() override { - return VARS_TO_STR9(pool_type, type_input, ne_input, k0, k1, s0, s1, p0, p1); - } - - test_pool2d(ggml_op_pool pool_type = GGML_OP_POOL_AVG, - ggml_type type_input = GGML_TYPE_F32, - std::array ne_input = {10, 10, 3, 1}, // [input_width, input_height, input_channels, 1] - int k0 = 3, int k1 = 3, - int s0 = 1, int s1 = 1, - int p0 = 1, int p1 = 1) - : pool_type(pool_type), type_input(type_input), ne_input(ne_input), k0(k0), k1(k1), s0(s0), s1(s1), p0(p0), p1(p1) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * input = ggml_new_tensor(ctx, type_input, 4, ne_input.data()); - ggml_set_param(input); - ggml_set_name(input, "input"); - - ggml_tensor * out = ggml_pool_2d(ctx, input, pool_type, k0, k1, s0, s1, p0, p1); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_POOL1D -struct test_pool1d : public test_case { - enum ggml_op_pool pool_type; - const ggml_type type_input; - const std::array ne_input; - const int k0; - const int s0; - const int p0; - - std::string vars() override { - return VARS_TO_STR6(pool_type, type_input, ne_input, k0, s0, p0); - } - - test_pool1d(ggml_op_pool pool_type = GGML_OP_POOL_AVG, - ggml_type type_input = GGML_TYPE_F32, - std::array ne_input = {10, 1, 1, 1}, - int k0 = 3, int s0 = 3, int p0 = 0) - : pool_type(pool_type), type_input(type_input), ne_input(ne_input), k0(k0), s0(s0), p0(p0) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * input = ggml_new_tensor(ctx, type_input, 4, ne_input.data()); - ggml_set_param(input); - ggml_set_name(input, "input"); - - ggml_tensor * out = ggml_pool_1d(ctx, input, pool_type, k0, s0, p0); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_CONV_TRANSPOSE_1D -struct test_conv_transpose_1d : public test_case { - const std::array ne_input; - const std::array ne_kernel; - - const int s0; // stride - const int p0; // padding - const int d0; // dilation - - std::string vars() override { - return VARS_TO_STR5(ne_input, ne_kernel, s0, p0, d0); - } - - test_conv_transpose_1d(std::array ne_input = {197, 32, 1, 1}, // [input_width, input_channels, 1 /* assert in cpu kernel*/, 1 (should be batch)] - std::array ne_kernel = {16, 32, 32, 1}, // [kernel_width, output_channels, input_channels, 1 (should be batch)] - int s0 = 1, int p0 = 0, int d0 = 1) - : ne_input(ne_input), ne_kernel(ne_kernel), s0(s0), p0(p0), d0(d0) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * input = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_input.data()); - ggml_set_name(input, "input"); - - ggml_tensor * kernel = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_kernel.data()); - ggml_set_name(kernel, "kernel"); - - ggml_tensor * out = ggml_conv_transpose_1d(ctx, kernel, input, s0, p0, d0); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_COL2IM_1D -struct test_col2im_1d : public test_case { - const ggml_type type; - const int64_t K; // kernel size - const int64_t OC; // output channels - const int64_t T_in; // input length (number of columns) - const int s0; // stride - const int p0; // padding cropped from both sides - - std::string vars() override { - return VARS_TO_STR6(type, K, OC, T_in, s0, p0); - } - - double max_nmse_err() override { - return type == GGML_TYPE_F32 ? 1e-7 : 5e-4; - } - - test_col2im_1d(ggml_type type = GGML_TYPE_F32, - int64_t K = 4, int64_t OC = 3, int64_t T_in = 7, - int s0 = 2, int p0 = 0) - : type(type), K(K), OC(OC), T_in(T_in), s0(s0), p0(p0) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * cols = ggml_new_tensor_2d(ctx, type, K*OC, T_in); - ggml_set_name(cols, "cols"); - - ggml_tensor * out = ggml_col2im_1d(ctx, cols, s0, (int) OC, p0); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_CONV_TRANSPOSE_2D -struct test_conv_transpose_2d : public test_case { - // Dimensions - const std::array ne_input; - const std::array ne_kernel; - const int stride; - // Types - const ggml_type kernel_type; - - std::string vars() override { - return VARS_TO_STR4(kernel_type, ne_input, ne_kernel, stride); - } - - double max_nmse_err() override { - return 5e-4; // The default 1e-7 is too small for Vulkan. - } - - test_conv_transpose_2d( - std::array ne_input = {10, 10, 3, 1}, // [input_width, input_height, input_channels, 1] - std::array ne_kernel = {3, 3, 3, 1}, // [kernel_width, kernel_height, input_channels, 1] - int stride = 1, - ggml_type kernel_type = GGML_TYPE_F16 - ) : ne_input(ne_input), ne_kernel(ne_kernel), stride(stride), kernel_type(kernel_type) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * input = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_input.data()); - ggml_set_name(input, "input"); - - ggml_tensor * kernel = ggml_new_tensor(ctx, kernel_type, 4, ne_kernel.data()); - ggml_set_name(kernel, "kernel"); - - ggml_tensor * out = ggml_conv_transpose_2d_p0(ctx, kernel, input, stride); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_IM2COL -struct test_im2col : public test_case { - const ggml_type type_input; - const ggml_type type_kernel; - const ggml_type dst_type; - const std::array ne_input; - const std::array ne_kernel; - // stride - const int s0; - const int s1; - // padding - const int p0; - const int p1; - // dilation - const int d0; - const int d1; - // mode - const bool is_2D; - - std::string vars() override { - return VARS_TO_STR12(type_input, type_kernel, dst_type, ne_input, ne_kernel, s0, s1, p0, p1, d0, d1, is_2D); - } - - test_im2col(ggml_type type_input = GGML_TYPE_F32, ggml_type type_kernel = GGML_TYPE_F16, ggml_type dst_type = GGML_TYPE_F32, - std::array ne_input = {10, 10, 3, 1}, // [input_width, input_height, input_channels, 1] - std::array ne_kernel = {3, 3, 3, 1}, // [kernel_width, kernel_height, input_channels, 1] - int s0 = 1, int s1 = 1, - int p0 = 1, int p1 = 1, - int d0 = 1, int d1 = 1, - bool is_2D = true) - : type_input(type_input), type_kernel(type_kernel), dst_type(dst_type), ne_input(ne_input), ne_kernel(ne_kernel), s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), is_2D(is_2D) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * input = ggml_new_tensor(ctx, type_input, 4, ne_input.data()); - ggml_set_param(input); - ggml_set_name(input, "input"); - - ggml_tensor * kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel.data()); - ggml_set_name(kernel, "kernel"); - - ggml_tensor * out = ggml_im2col(ctx, kernel, input, s0, s1, p0, p1, d0, d1, is_2D, dst_type); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_IM2COL_3D -struct test_im2col_3d : public test_case { - const ggml_type type_input; - const ggml_type type_kernel; - const ggml_type dst_type; - const std::array ne_input; - const std::array ne_kernel; - // stride - const int s0; - const int s1; - const int s2; - // padding - const int p0; - const int p1; - const int p2; - // dilation - const int d0; - const int d1; - const int d2; - - const int64_t IC; - const bool v; - - std::string vars() override { - return VARS_TO_STR16(type_input, type_kernel, dst_type, ne_input, ne_kernel, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, v); - } - - test_im2col_3d(ggml_type type_input = GGML_TYPE_F32, ggml_type type_kernel = GGML_TYPE_F16, ggml_type dst_type = GGML_TYPE_F32, - std::array ne_input = {10, 10, 10, 9}, // [OC*IC, KD, KH, KW] - std::array ne_kernel = {3, 3, 3, 1}, // [N*IC, ID, IH, IW] - int64_t IC = 3, - int s0 = 1, int s1 = 1, int s2 = 1, - int p0 = 1, int p1 = 1, int p2 = 1, - int d0 = 1, int d1 = 1, int d2 = 1, - bool v = false) - : type_input(type_input), type_kernel(type_kernel), dst_type(dst_type), ne_input(ne_input), ne_kernel(ne_kernel), s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), IC(IC), v(v) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * input = ggml_new_tensor(ctx, type_input, 4, ne_input.data()); - ggml_set_param(input); - ggml_set_name(input, "input"); - - if (v) { - input = ggml_view_4d(ctx, input, ne_input[0] - 2, ne_input[1] - 2, ne_input[2] - 2, ne_input[3] - 2, input->nb[1], input->nb[2], input->nb[3], 0); - ggml_set_name(input, "view_of_input"); - } - - ggml_tensor * kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel.data()); - ggml_set_name(kernel, "kernel"); - - ggml_tensor * out = ggml_im2col_3d(ctx, kernel, input, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, dst_type); - ggml_set_name(out, "out"); - - return out; - } -}; - -// CONV_2D -struct test_conv_2d : public test_case { - const std::array ne_input; - const std::array ne_kernel; - const ggml_type type_kernel; - const int stride0; - const int stride1; - const int padding0; - const int padding1; - const int dilation0; - const int dilation1; - // Whether the inputs are contiguous in the channel dim or the width dim - const bool cwhn; - - // If true, the direct CONV_2D will be used in the graph, otherwise it - // uses ggml_conv_2d: - // * if the program is called with -o CONV_2D_DIRECT_IMPL, the - // CONV_2D graph will be built, while - // * if the program is called with -o CONV_2D_INDIRECT_IMPL, the - // IM2COL -> MUL_MM graph will be built. - - std::string vars() override { - return VARS_TO_STR10(ne_input, ne_kernel, type_kernel, stride0, stride1, padding0, padding1, dilation0, dilation1, cwhn); - } - - double max_nmse_err() override { - return 5e-4; - } - - uint64_t op_flops(ggml_tensor * t) override { - GGML_UNUSED(t); - // Just counting matmul costs: - // KxCRS @ CRSxNPQ = KxNPQ --> KxNPQx(CRS+CRS-1) flops - - // Copied from ggml.c: int64_t ggml_calc_conv_output_size(int64_t ins, int64_t ks, int s, int p, int d) - auto calc_conv_output_size = [](int64_t ins, int64_t ks, int s, int p, int d) -> int64_t { - return (ins + 2 * p - d * (ks - 1) - 1) / s + 1; - }; - - int64_t W = ne_input[0]; - int64_t H = ne_input[1]; - int64_t KW = ne_kernel[0]; - int64_t KH = ne_kernel[1]; - int64_t Cin = ne_kernel[2]; - int64_t Cout = ne_kernel[3]; - int64_t N = ne_input[3]; - int64_t OH = calc_conv_output_size(H, KH, stride0, padding0, dilation0); - int64_t OW = calc_conv_output_size(W, KW, stride0, padding0, dilation0); - - int64_t K = Cout; - int64_t CRS = Cin * KH * KW; - int64_t NPQ = N * OH * OW; - - return K * NPQ * (2 * CRS - 1); - } - - test_conv_2d(std::array ne_input = { 64, 64, 16, 1 }, - std::array ne_kernel = { 3, 3, 1, 16 }, ggml_type type_kernel = GGML_TYPE_F32, int stride0 = 1, - int stride1 = 1, int padding0 = 0, int padding1 = 0, int dilation0 = 1, int dilation1 = 1, bool cwhn = false) : - ne_input(ne_input), - ne_kernel(ne_kernel), - type_kernel(type_kernel), - stride0(stride0), - stride1(stride1), - padding0(padding0), - padding1(padding1), - dilation0(dilation0), - dilation1(dilation1), - cwhn(cwhn) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * input = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_input.data()); - ggml_set_name(input, "input"); - - ggml_tensor * kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel.data()); - ggml_set_name(kernel, "kernel"); - - if (cwhn) { - // change memory layout to channel-most-contiguous (CWHN), - // then permute it back so NE matches the original input - input = ggml_cont(ctx, ggml_permute(ctx, input, 1, 2, 0, 3)); - input = ggml_permute(ctx, input, 2, 0, 1, 3); - kernel = ggml_cont(ctx, ggml_permute(ctx, kernel, 2, 3, 1, 0)); - kernel = ggml_permute(ctx, kernel, 3, 2, 0, 1); - } - - ggml_tensor * out = - ggml_conv_2d_direct(ctx, kernel, input, stride0, stride1, padding0, padding1, dilation0, dilation1); - ggml_set_name(out, "out"); - return out; - } -}; - -// GGML_OP_CONV_2D_DW -struct test_conv_2d_dw : public test_case { - const std::array ne_input; - const std::array ne_kernel; - const ggml_type type_kernel; - const int stride; - const int padding; - const int dilation; - const bool cwhn; - - std::string vars() override { - return VARS_TO_STR7(ne_input, ne_kernel, type_kernel, stride, padding, dilation, cwhn); - } - - test_conv_2d_dw( - std::array ne_input = {64, 64, 16, 1}, - std::array ne_kernel = {3, 3, 1, 16}, - ggml_type type_kernel = GGML_TYPE_F32, - int stride = 1, int padding = 0, int dilation = 1, bool cwhn = false) - : ne_input(ne_input), ne_kernel(ne_kernel), type_kernel(type_kernel), stride(stride), padding(padding), dilation(dilation), cwhn(cwhn) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * input = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_input.data()); - ggml_set_name(input, "input"); - - ggml_tensor * kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel.data()); - ggml_set_name(kernel, "kernel"); - - if (cwhn) { - // change memory layout to channel-most-contiguous (CWHN), - // then permute it back so NE matches the original input - input = ggml_cont(ctx, ggml_permute(ctx, input, 1, 2, 0, 3)); - input = ggml_permute(ctx, input, 2, 0, 1, 3); - kernel = ggml_cont(ctx, ggml_permute(ctx, kernel, 2, 3, 1, 0)); - kernel = ggml_permute(ctx, kernel, 3, 2, 0, 1); - } - - ggml_tensor * out = ggml_conv_2d_dw_direct( - ctx, kernel, input, - stride, stride, padding, padding, dilation, dilation); - ggml_set_name(out, "out"); - return out; - } -}; - -// GGML_OP_CONV_3D -struct test_conv_3d : public test_case { - // Logical 5D dimensions - const int64_t N, IC, ID, IH, IW; - const int64_t OC, KD, KH, KW; - // Conv params - const int s0, s1, s2; - const int p0, p1, p2; - const int d0, d1, d2; - // Types - const ggml_type type_kernel; - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "CONV_3D"; - } - - std::string vars() override { - return VARS_TO_STR11(N, IC, ID, IH, IW, OC, KD, KH, KW, s0, s1) + "," + - VARS_TO_STR8(s2, p0, p1, p2, d0, d1, d2, type_kernel); - } - - double max_nmse_err() override { - return 5e-4; - } - - uint64_t op_flops(ggml_tensor * t) override { - GGML_UNUSED(t); - auto calc_conv_output_size = [](int64_t ins, int64_t ks, int s, int p, int d) -> int64_t { - return (ins + 2 * p - d * (ks - 1) - 1) / s + 1; - }; - const int64_t OD = calc_conv_output_size(ID, KD, s2, p2, d2); - const int64_t OH = calc_conv_output_size(IH, KH, s1, p1, d1); - const int64_t OW = calc_conv_output_size(IW, KW, s0, p0, d0); - - return (uint64_t)N * OC * OD * OH * OW * (2 * IC * KD * KH * KW - 1); - } - - test_conv_3d( - int64_t N, int64_t IC, int64_t ID, int64_t IH, int64_t IW, - int64_t OC, int64_t KD, int64_t KH, int64_t KW, - int s0, int s1, int s2, - int p0, int p1, int p2, - int d0, int d1, int d2, - ggml_type type_kernel - ) : N(N), IC(IC), ID(ID), IH(IH), IW(IW), - OC(OC), KD(KD), KH(KH), KW(KW), - s0(s0), s1(s1), s2(s2), - p0(p0), p1(p1), p2(p2), - d0(d0), d1(d1), d2(d2), - type_kernel(type_kernel) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - // GGML input tensor is packed as [W, H, D, C*N] - const int64_t ne_input[] = {IW, IH, ID, IC * N}; - ggml_tensor * input = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_input); - ggml_set_name(input, "input"); - - // GGML kernel tensor is packed as [KW, KH, KD, IC*OC] - const int64_t ne_kernel[] = {KW, KH, KD, IC * OC}; - ggml_tensor * kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel); - ggml_set_name(kernel, "kernel"); - - ggml_tensor * out = ggml_conv_3d_direct(ctx, kernel, input, s0, s1, s2, p0, p1, p2, d0, d1, d2, (int)IC, (int)N, (int)OC); - ggml_set_name(out, "out"); - return out; - } -}; - -// GGML_OP_CONCAT -struct test_concat : public test_case { - const ggml_type type; - const std::array ne_a; - const int64_t ne_b_d; - const int dim; - const int v; // view (1 << 0: non-cont a, 1 << 1: non-cont b) - - std::string vars() override { - return VARS_TO_STR5(type, ne_a, ne_b_d, dim, v); - } - - test_concat(ggml_type type = GGML_TYPE_F32, - std::array ne_a = {10, 5, 5, 5}, - int64_t ne_b_d = 5, - int dim = 2, int v = 0) - : type(type), ne_a(ne_a), ne_b_d(ne_b_d), dim(dim), v(v) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - auto ne_b = ne_a; - ne_b[dim] = ne_b_d; - ggml_tensor * a; - if (v & 1) { - auto ne = ne_a; ne[0] *= 2; ne[1] *= 4; ne[2] *= 3; - a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); - ggml_set_name(a, "view_of_a"); - } else { - a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_set_name(a, "a"); - } - ggml_tensor * b; - if (v & 2) { - auto ne = ne_b; ne[0] *= 3; ne[1] *= 2; ne[2] *= 4; - b = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(b, "b"); - - b = ggml_view_4d(ctx, b, ne_b[0], ne_b[1], ne_b[2], ne_b[3], b->nb[1], b->nb[2], b->nb[3], 0); - ggml_set_name(b, "view_of_b"); - } else { - b = ggml_new_tensor(ctx, type, 4, ne_b.data()); - ggml_set_name(b, "b"); - } - - ggml_tensor * out = ggml_concat(ctx, a, b, dim); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_ARGSORT -struct test_argsort : public test_case { - const ggml_type type; - const std::array ne; - ggml_sort_order order; - - std::string vars() override { - return VARS_TO_STR3(type, ne, order); - } - - test_argsort(ggml_type type = GGML_TYPE_F32, - std::array ne = {16, 10, 10, 10}, - ggml_sort_order order = GGML_SORT_ORDER_ASC) - : type(type), ne(ne), order(order) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_argsort(ctx, a, order); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - std::random_device rd; - std::default_random_engine rng(rd()); - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I32) { - // indices - std::vector data(ggml_nelements(t)); - for (int i = 0; i < ggml_nelements(t); i++) { - data[i] = rand(); - } - std::shuffle(data.begin(), data.end(), rng); - ggml_backend_tensor_set(t, data.data(), 0, ne[0]*ne[1]*ne[2]*ne[3] * sizeof(int)); - } else if (t->type == GGML_TYPE_F32) { - // initialize with unique values to avoid ties - for (int64_t r = 0; r < ggml_nrows(t); r++) { - std::vector data(t->ne[0]); - for (int i = 0; i < t->ne[0]; i++) { - data[i] = i; - } - std::shuffle(data.begin(), data.end(), rng); - ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(float)); - } - } else { - GGML_ABORT("fatal error"); - } - } - } -}; - -// GGML_OP_TOP_K -struct test_top_k : public test_case { - const ggml_type type; - const std::array ne; - const int k; - const bool ties; - ggml_tensor * input {}; - - std::string vars() override { - return VARS_TO_STR4(type, ne, k, ties); - } - - test_top_k(ggml_type type = GGML_TYPE_F32, - std::array ne = {16, 10, 10, 10}, - int k = 4, bool ties = false) - : type(type), ne(ne), k(k), ties(ties) {} - - double max_err() override { - return 0.0; - } - - // When there are ties, only validate the final result. - // The logic in err can't handle the sentinel tensors. - bool run_whole_graph() override { return ties; } - - double err(const float * a, const float * b, size_t n) override { - // When there are no ties, we expect the exact same set of indices, - // but possibly in a different order. When there are ties, the indices - // can be different but the input values they correspond to should be - // the same. The logic for ties could work for non-ties, but only for - // the output tensor, not for the sentinel tensors. - if (ties) { - std::vector src(ggml_nelements(input)); - - ggml_backend_tensor_get(input, src.data(), 0, ggml_nelements(input) * ggml_type_size(type)); - - double diff = 0.0f; - - GGML_ASSERT(n == (size_t)(ggml_nrows(input) * k)); - int64_t cols = input->ne[0]; - std::vector ia(k); - std::vector ib(k); - std::vector asrc(k); - std::vector bsrc(k); - for (int64_t r = 0; r < ggml_nrows(input); r++) { - // Convert indices for the row back to integer - for (int64_t c = 0; c < k; c++) { - ia[c] = (int32_t)a[r * k + c]; - ib[c] = (int32_t)b[r * k + c]; - } - // The src values for each row should match. - for (int64_t c = 0; c < k; c++) { - asrc[c] = src[r * cols + ia[c]]; - bsrc[c] = src[r * cols + ib[c]]; - } - diff += jdst(asrc.data(), bsrc.data(), k); - // There should be no duplicate indices - std::sort(ia.begin(), ia.end()); - std::sort(ib.begin(), ib.end()); - if (std::adjacent_find(ia.begin(), ia.end()) != ia.end()) { - diff += 1; - } - if (std::adjacent_find(ib.begin(), ib.end()) != ib.end()) { - diff += 1; - } - } - return diff; - } else { - std::vector ia(n); - std::vector ib(n); - - double diff = 0.0f; - - for (size_t i = 0; i < n; i++) { - ia[i] = (int32_t) a[i]; - ib[i] = (int32_t) b[i]; - - // penalize the result if the data is not integer valued - diff += std::fabs(a[i] - ia[i]); - diff += std::fabs(b[i] - ib[i]); - } - - return diff + jdst(ia.data(), ib.data(), n); - } - } - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - // Save 'a' for err() - input = a; - - ggml_tensor * out = ggml_top_k(ctx, a, k); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - std::random_device rd; - std::default_random_engine rng(rd()); - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - int tie_denom = std::max(1, std::min(10, k / 2)); - for (int64_t r = 0; r < ggml_nrows(t); r++) { - std::vector data(t->ne[0]); - for (int i = 0; i < t->ne[0]; i++) { - if (ties) { - // integer division to introduce duplicates - data[i] = i / tie_denom; - } else { - data[i] = i; - } - } - std::shuffle(data.begin(), data.end(), rng); - ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(float)); - } - } - } -}; - -enum MoeGatingFunc { - GATING_FUNC_SOFTMAX, - GATING_FUNC_SIGMOID, - GATING_FUNC_SOFTMAX_WEIGHT, -}; - -struct test_topk_moe : public test_case { - const std::array ne; - const int n_expert_used; - const bool with_norm; - const bool bias_probs; - const MoeGatingFunc gating_func; - const float scale_w; - ggml_tensor * weights {}; - ggml_tensor * selected_experts {}; - - test_topk_moe(std::array ne = { 10, 5, 1, 1 }, - int n_expert_used = 1, - bool with_norm = false, - bool bias_probs = false, - MoeGatingFunc gating_func = GATING_FUNC_SOFTMAX, - float scale_w = 0.0f) : - ne(ne), - n_expert_used(n_expert_used), - with_norm(with_norm), - bias_probs(bias_probs), - gating_func(gating_func), - scale_w(scale_w) { - GGML_ASSERT(n_expert_used <= ne[0]); - } - - std::string vars() override { return VARS_TO_STR6(ne, n_expert_used, with_norm, bias_probs, gating_func, scale_w); } - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "TOPK_MOE"; - } - - bool run_whole_graph() override { return true; } - - ggml_tensor * build_graph(ggml_context * ctx) override { - const int n_expert = ne[0]; - const int n_tokens = ne[1]; - - ggml_tensor * logits = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data()); - ggml_tensor * probs = - (gating_func == GATING_FUNC_SOFTMAX) ? ggml_soft_max(ctx, logits) : - (gating_func == GATING_FUNC_SIGMOID) ? ggml_sigmoid(ctx, logits) : logits; - ggml_set_name(probs, "probs"); - - ggml_tensor * selection_probs = probs; - if (bias_probs) { - ggml_tensor * exp_probs_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]); - ggml_set_name(exp_probs_b, "exp_probs_b"); - selection_probs = ggml_add(ctx, probs, exp_probs_b); - ggml_set_name(selection_probs, "selection_probs"); - } - - selected_experts = ggml_argsort_top_k(ctx, selection_probs, n_expert_used); // [n_expert_used, n_tokens] - ggml_set_name(selected_experts, "selected_experts"); - - weights = ggml_get_rows(ctx, ggml_reshape_3d(ctx, probs, 1, n_expert, n_tokens), selected_experts); // [1, n_expert_used, n_tokens] - ggml_set_name(weights, "weights"); - - if (gating_func == GATING_FUNC_SOFTMAX_WEIGHT) { - weights = ggml_reshape_2d(ctx, weights, n_expert_used, n_tokens); - weights = ggml_soft_max(ctx, weights); // [n_expert_used, n_tokens] - weights = ggml_reshape_3d(ctx, weights, 1, n_expert_used, n_tokens); - } - - if (with_norm) { - weights = ggml_reshape_2d(ctx, weights, n_expert_used, n_tokens); - ggml_tensor * weights_sum = ggml_sum_rows(ctx, weights); // [1, n_tokens] - ggml_set_name(weights_sum, "weights_sum"); - - weights_sum = ggml_clamp(ctx, weights_sum, 6.103515625e-5, INFINITY); - weights = ggml_div(ctx, weights, weights_sum); // [n_expert_used, n_tokens] - weights = ggml_reshape_3d(ctx, weights, 1, n_expert_used, n_tokens); - } - - if (scale_w) { - weights = ggml_scale(ctx, weights, scale_w); - } - - ggml_set_name(weights, "weights"); - return weights; - } - // Verify two outputs - std::vector fusion_test_nodes() override { return { selected_experts, weights }; } - - // allow output in arbitrary order - double err(const float * a, const float * b, size_t n) override { - std::vector a2(n); - std::vector b2(n); - for (size_t i = 0; i < n; ++i) { - a2[i] = a[i]; - b2[i] = b[i]; - } - std::sort(a2.begin(), a2.end()); - std::sort(b2.begin(), b2.end()); - return nmse(a2.data(), b2.data(), n); - } -}; - -struct test_mul_mat_vec_fusion : public test_case { - const ggml_type type; - const ggml_glu_op glu_op; - const int64_t m; - const int64_t n; - const int64_t k; - const bool use_id; - const int n_mats; - const int n_used; - const bool b; // broadcast b matrix (only for use_id) - const bool with_bias; - const bool with_gate; - const bool with_lane_scale; - std::array batch_dims; - - test_mul_mat_vec_fusion(ggml_type type, ggml_glu_op op, int64_t m, int64_t n, int64_t k, - bool use_id = false, int n_mats = 1, int n_used = 1, bool b = false, bool with_bias = false, bool with_gate = true, - bool with_lane_scale = false, std::array batch_dims = {4, 2}) - : type(type), glu_op(op), m(m), n(n), k(k), use_id(use_id), n_mats(n_mats), n_used(n_used), b(b), with_bias(with_bias), - with_gate(with_gate), with_lane_scale(with_lane_scale), batch_dims(batch_dims) { - if (use_id) { - GGML_ASSERT(n_used <= n_mats); - } - } - - std::string vars() override { - return VARS_TO_STR13(type, glu_op, m, n, k, use_id, n_mats, n_used, b, with_bias, with_gate, with_lane_scale, batch_dims); - } - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "MUL_MAT_VEC_FUSION"; - } - - bool run_whole_graph() override { return true; } - bool use_weight_context() override { return use_id && with_lane_scale; } - - ggml_tensor * build_gate(ggml_context * ctx, ggml_tensor * ffn_gate, ggml_tensor * ffn_up) { - ggml_tensor * out = nullptr; - if (with_gate) { - if (glu_op == GGML_GLU_OP_SWIGLU_OAI) { - constexpr float alpha = 1.702f; - constexpr float limit = 7.0f; - out = ggml_swiglu_oai(ctx, ffn_gate, ffn_up, alpha, limit); - } else { - out = ggml_glu_split(ctx, ffn_gate, ffn_up, glu_op); - } - } - return out; - } - - ggml_tensor * build_lane_scale_dense(ggml_context * ctx, ggml_tensor * out) { - ggml_tensor * scale = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); - return ggml_mul(ctx, out, scale); - } - - ggml_tensor * build_lane_scale_id(ggml_context * ctx, ggml_context * ctx_weights, ggml_tensor * out, ggml_tensor * ids) { - GGML_ASSERT(ctx_weights); - ggml_tensor * scale = ggml_new_tensor_1d(ctx_weights, GGML_TYPE_F32, n_mats); - ggml_tensor * s = ggml_reshape_3d(ctx, scale, 1, n_mats, 1); - s = ggml_repeat_4d(ctx, s, 1, n_mats, m, 1); - s = ggml_get_rows(ctx, s, ids); - return ggml_mul(ctx, out, s); - } - - ggml_tensor * build_graph(ggml_context * ctx) override { - GGML_ASSERT(!use_weight_context()); - return build_graph(ctx, nullptr); - } - - ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { - if (!use_id) { - const int channels = batch_dims[0]; - const int samples = batch_dims[1]; - std::array ne = { k, m, channels, samples }; - std::array ne0 = { k, n, channels, samples }; - - ggml_tensor * cur = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data()); - ggml_tensor * gate = with_gate ? ggml_new_tensor(ctx, type, 4, ne0.data()) : nullptr; - ggml_tensor * up = ggml_new_tensor(ctx, type, 4, ne0.data()); - - auto build_lane_up = [&]() { - ggml_tensor * ffn_up = ggml_mul_mat(ctx, up, cur); - if (with_lane_scale) { - ffn_up = build_lane_scale_dense(ctx, ffn_up); - } - if (with_bias) { - std::array bias_ne = { ffn_up->ne[0], 1, channels, samples }; - ggml_tensor * up_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data()); - ffn_up = ggml_add(ctx, ffn_up, up_bias); - } - return ffn_up; - }; - - auto build_lane_gate = [&]() { - ggml_tensor * ffn_gate = ggml_mul_mat(ctx, gate, cur); - if (with_lane_scale) { - ffn_gate = build_lane_scale_dense(ctx, ffn_gate); - } - if (with_bias) { - std::array bias_ne = { ffn_gate->ne[0], 1, channels, samples }; - ggml_tensor * gate_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data()); - ffn_gate = ggml_add(ctx, ffn_gate, gate_bias); - } - return ffn_gate; - }; - - ggml_tensor * ffn_up = build_lane_up(); - ggml_tensor * ffn_gate = with_gate ? build_lane_gate() : nullptr; - - ggml_tensor * out = with_gate ? build_gate(ctx, ffn_gate, ffn_up) : ffn_up; - - std::array bias2_ne = { out->ne[0], 1, channels, samples }; - ggml_tensor * bias2 = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias2_ne.data()); - out = ggml_add(ctx, out, bias2); - - ggml_set_name(out, "out"); - return out; - } else { - ggml_tensor * gates = ggml_new_tensor_3d(ctx, type, k, n, n_mats); - ggml_tensor * ups = ggml_new_tensor_3d(ctx, type, k, n, n_mats); - ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, m); - - if (n_used != n_mats) { - ids = ggml_view_2d(ctx, ids, n_used, m, ids->nb[1], 0); - } - - ggml_tensor * cur = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, this->b ? 1 : n_used, m); - ggml_set_name(cur, "cur"); - - auto build_lane_up = [&]() { - ggml_tensor * ffn_up = ggml_mul_mat_id(ctx, ups, cur, ids); - if (with_lane_scale) { - ffn_up = build_lane_scale_id(ctx, ctx_weights, ffn_up, ids); - } - if (with_bias) { - ggml_tensor * up_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_up->ne[0], n_mats); - ffn_up = ggml_add_id(ctx, ffn_up, up_bias_param, ids); - } - return ffn_up; - }; - - auto build_lane_gate = [&]() { - ggml_tensor * ffn_gate = ggml_mul_mat_id(ctx, gates, cur, ids); - if (with_lane_scale) { - ffn_gate = build_lane_scale_id(ctx, ctx_weights, ffn_gate, ids); - } - if (with_bias) { - ggml_tensor * gate_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_gate->ne[0], n_mats); - ffn_gate = ggml_add_id(ctx, ffn_gate, gate_bias_param, ids); - } - return ffn_gate; - }; - - ggml_tensor * ffn_up = build_lane_up(); - ggml_tensor * ffn_gate = with_gate ? build_lane_gate() : nullptr; - - ggml_tensor * out = with_gate ? build_gate(ctx, ffn_gate, ffn_up) : ffn_up; - - std::array scale_ne { 1, out->ne[1], out->ne[2], out->ne[3] }; - ggml_tensor * scale = ggml_new_tensor(ctx, out->type, 4, scale_ne.data()); - out = ggml_mul(ctx, out, scale); - - ggml_set_name(out, "out"); - return out; - } - } - - void initialize_tensors(ggml_context * ctx) override { - if (!use_id) { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t); - } - } else { - init_mul_mat_id_tensors(ctx, n_mats); - } - } - - double max_nmse_err() override { - return 5e-3; - } -}; - -// GGML_OP_SUM -struct test_sum : public test_case { - const ggml_type type; - const std::array ne; - const std::array permute; - bool _use_permute; - - std::string vars() override { - std::string v = VARS_TO_STR2(type, ne); - if (_use_permute) v += "," + VAR_TO_STR(permute); - return v; - } - - test_sum(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}, - std::array permute = {0, 0, 0, 0}) - : type(type), ne(ne), permute(permute), - _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - if (_use_permute) { - a = ggml_permute(ctx, a, permute[0], permute[1], permute[2], permute[3]); - ggml_set_name(a, "a_permuted"); - } - - ggml_tensor * out = ggml_sum(ctx, a); - ggml_set_name(out, "out"); - - return out; - } - - float grad_eps() override { - return 0.1f * sqrtf(ne[0]*ne[1]*ne[2]*ne[3]); - } - - // Don't center the distribution around zero. Helps to avoid catastrophic cancellation. - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -0.9f, 1.1f); - } - } -}; - -// GGML_OP_SUM_ROWS -struct test_sum_rows : public test_case { - const ggml_type type; - const std::array ne; - const bool permute; - const bool slice; - - std::string vars() override { - return VARS_TO_STR4(type, ne, permute, slice); - } - - test_sum_rows(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}, - bool permute = false, bool slice = false) - : type(type), ne(ne), permute(permute), slice(slice) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - if (slice) { - a = ggml_view_4d(ctx, a, - ne[0], ne[1], ne[2] / 2, ne[3] - 1, - a->nb[1], a->nb[2] * 2, a->nb[3], /*offset=*/a->nb[3]); - } - if (permute) { - a = ggml_permute(ctx, a, 0, 2, 3, 1); - } - - ggml_tensor * out = ggml_sum_rows(ctx, a); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_MEAN -struct test_mean : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_mean(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_mean(ctx, a); - ggml_set_name(out, "out"); - - return out; - } - - float grad_eps() override { - return 0.1f * ne[0]*ne[1]*ne[2]*ne[3]; - } - - // Don't center the distribution around zero. Helps to avoid catastrophic cancellation. - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -0.9f, 1.1f); - } - } -}; - -// GGML_OP_UPSCALE -struct test_upscale : public test_case { - const ggml_type type; - const std::array ne; - const int32_t scale_factor; - const bool transpose; - const ggml_scale_mode mode; - - std::string vars() override { - return VARS_TO_STR5(type, ne, scale_factor, mode, transpose); - } - - test_upscale(ggml_type type = GGML_TYPE_F32, - std::array ne = {512, 512, 3, 1}, - int32_t scale_factor = 2, ggml_scale_mode mode = GGML_SCALE_MODE_NEAREST, bool transpose = false) - : type(type), ne(ne), scale_factor(scale_factor), transpose(transpose), mode(mode) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - if (transpose) { - a = ggml_transpose(ctx, a); - ggml_set_name(a, "a_transposed"); - } - - ggml_tensor * out = ggml_upscale(ctx, a, scale_factor, mode); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_UPSCALE (via ggml_interpolate) -struct test_interpolate : public test_case { - const ggml_type type; - const std::array ne; - const std::array ne_tgt; - const ggml_scale_mode mode = GGML_SCALE_MODE_NEAREST; - - std::string vars() override { - return VARS_TO_STR4(type, ne, ne_tgt, mode); - } - - test_interpolate(ggml_type type = GGML_TYPE_F32, - std::array ne = {2, 5, 7, 11}, - std::array ne_tgt = {5, 7, 11, 13}, - ggml_scale_mode mode = GGML_SCALE_MODE_NEAREST) - : type(type), ne(ne), ne_tgt(ne_tgt), mode(mode) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_interpolate(ctx, a, ne_tgt[0], ne_tgt[1],ne_tgt[2], ne_tgt[3], mode); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_GROUP_NORM -struct test_group_norm : public test_case { - const ggml_type type; - const std::array ne; - const int32_t num_groups; - const float eps; - - std::string vars() override { - return VARS_TO_STR4(type, ne, num_groups, eps); - } - - test_group_norm(ggml_type type = GGML_TYPE_F32, - std::array ne = {64, 64, 320, 1}, - int32_t num_groups = 32, - float eps = 1e-6f) - : type(type), ne(ne), num_groups(num_groups), eps(eps) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_group_norm(ctx, a, num_groups, eps); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_GROUP_NORM + GGML_OP_MUL + GGML_OP_ADD -struct test_group_norm_mul_add : public test_case { - const ggml_type type; - const std::array ne; - int num_groups; - float eps; - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "GROUP_NORM_MUL_ADD"; - } - - bool run_whole_graph() override { return true; } - - std::string vars() override { - return VARS_TO_STR4(type, ne, num_groups, eps); - } - - test_group_norm_mul_add(ggml_type type = GGML_TYPE_F32, - std::array ne = {128, 1, 1, 1}, - int num_groups = 4, - float eps = 1e-5f) - : type(type), ne(ne), num_groups(num_groups), eps(eps) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_tensor * w = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); ggml_set_param(w); ggml_set_param(b); - ggml_set_name(a, "a"); ggml_set_name(w, "w"); ggml_set_name(b, "b"); - ggml_tensor * n = ggml_group_norm(ctx, a, num_groups, eps); - ggml_tensor * m = ggml_mul(ctx, n, w); - ggml_tensor * out = ggml_add(ctx, m, b); - ggml_set_name(out, "out"); - return out; - } -}; - -// GGML_OP_L2_NORM -struct test_l2_norm : public test_case { - const ggml_type type; - const std::array ne; - const float eps; - bool v; - bool noncontig_rows; - - std::string vars() override { - return VARS_TO_STR5(type, ne, eps, v, noncontig_rows); - } - - test_l2_norm(ggml_type type = GGML_TYPE_F32, - std::array ne = {64, 64, 320, 1}, - float eps = 1e-12f, - bool v = false, - bool noncontig_rows = false) - : type(type), ne(ne), eps(eps), v(v), noncontig_rows(noncontig_rows) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - const std::array ne_a = noncontig_rows ? - std::array{ ne[1], ne[0], ne[2], ne[3] } : ne; - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_set_name(a, "a"); - - if (noncontig_rows) { - a = ggml_permute(ctx, a, 1, 0, 2, 3); - ggml_set_name(a, "permuted a"); - } - if (v) { - a = ggml_view_4d(ctx, a, a->ne[0]/2, a->ne[1]/2, a->ne[2]/2, a->ne[3]/2, a->nb[1], a->nb[2], a->nb[3], 0); - ggml_set_name(a, "view of a"); - } - - ggml_tensor * out = ggml_l2_norm(ctx, a, eps); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_ACC -struct test_acc : public test_case { - const ggml_type type; - const std::array ne_a; - const std::array ne_b; - const int64_t stride_dim; - - std::string vars() override { - return VARS_TO_STR4(type, ne_a, ne_b, stride_dim); - } - - test_acc(ggml_type type = GGML_TYPE_F32, - std::array ne_a = {256, 17, 2, 3}, - std::array ne_b = {256, 16, 2, 3}, - uint64_t stride_dim = -1) - : type(type), ne_a(ne_a), ne_b(ne_b), stride_dim(stride_dim) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * b; - if (stride_dim == 1 || stride_dim == 2 || stride_dim == 3) { - // Create a larger tensor and take a view at a non-zero offset. - // This tests that the backend correctly handles b's data offset - std::array ne_b_pad = {ne_b[0], ne_b[1], ne_b[2], ne_b[3]}; - ne_b_pad[stride_dim] += 1; - ggml_tensor * b_pad = ggml_new_tensor(ctx, type, 4, ne_b_pad.data()); - ggml_set_param(b_pad); - ggml_set_name(b_pad, "b_pad"); - // View that skips the first row, so b has a non-zero byte offset - b = ggml_view_4d(ctx, b_pad, - ne_b[0], ne_b[1], ne_b[2], ne_b[3], - b_pad->nb[1], b_pad->nb[2], b_pad->nb[3], - b_pad->nb[1]); - } else { - b = ggml_new_tensor(ctx, type, 4, ne_b.data()); - ggml_set_param(b); - } - ggml_set_name(b, "b"); - - // When ne_b[0] < ne_a[0], a->nb[1] != b->nb[1], so the stride - // parameters to ggml_acc don't match b's natural stride. - ggml_tensor * out = ggml_acc(ctx, a, b, a->nb[1], a->nb[2], a->nb[3], 0); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_PAD -struct test_pad : public test_case { - const ggml_type type; - const std::array ne_a; - const int pad_0; - const int pad_1; - const bool circular; - - std::string vars() override { - return VARS_TO_STR5(type, ne_a, pad_0, pad_1, circular); - } - - test_pad(ggml_type type = GGML_TYPE_F32, - std::array ne_a = {512, 512, 1, 1}, - int pad_0 = 1, int pad_1 = 1, bool circular = false) - : type(type), ne_a(ne_a), pad_0(pad_0), pad_1(pad_1), circular(circular) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_set_name(a, "a"); - - ggml_tensor * out = circular - ? ggml_pad_circular(ctx, a, pad_0, pad_1, 0, 0) - : ggml_pad(ctx, a, pad_0, pad_1, 0, 0); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_PAD (with extension) -struct test_pad_ext : public test_case { - const ggml_type type; - const std::array ne_a; - const int lp0; - const int rp0; - const int lp1; - const int rp1; - const int lp2; - const int rp2; - const int lp3; - const int rp3; - const int tfrm; // 0 - none, 1 - non-cont, 2 - perm - const bool circular; - - std::string vars() override { - return VARS_TO_STR12(type, ne_a, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3, tfrm, circular); - } - - test_pad_ext(ggml_type type = GGML_TYPE_F32, - std::array ne_a = {512, 512, 3, 1}, - int lp0 = 1, int rp0 = 1, int lp1 = 1, int rp1 = 1, - int lp2 = 1, int rp2 = 1, int lp3 = 1, int rp3 = 1, - int tfrm = 0, bool circular = false) - : type(type), ne_a(ne_a), lp0(lp0), rp0(rp0), lp1(lp1), rp1(rp1), lp2(lp2), rp2(rp2), lp3(lp3), rp3(rp3), - tfrm(tfrm), circular(circular) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_set_name(a, "a"); - - if (tfrm == 1) { - a = ggml_view_4d(ctx, a, (a->ne[0] + 1) / 2, (a->ne[1] + 1) / 2, (a->ne[2] + 1) / 2, (a->ne[3] + 1) / 2, a->nb[1], a->nb[2], a->nb[3], 0); - ggml_set_name(a, "view of a"); - } else if (tfrm == 2) { - a = ggml_permute(ctx, a, 2, 1, 0, 3); - ggml_set_name(a, "permuted a"); - } - - ggml_tensor * out = circular - ? ggml_pad_ext_circular(ctx, a, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3) - : ggml_pad_ext (ctx, a, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_PAD_REFLECT_1D -struct test_pad_reflect_1d : public test_case { - const ggml_type type; - const std::array ne_a; - const int pad_0; - const int pad_1; - - std::string vars() override { - return VARS_TO_STR4(type, ne_a, pad_0, pad_1); - } - - test_pad_reflect_1d(ggml_type type = GGML_TYPE_F32, - std::array ne_a = {512, 34, 2, 1}, - int pad_0 = 10, int pad_1 = 9) - : type(type), ne_a(ne_a), pad_0(pad_0), pad_1(pad_1) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 2, ne_a.data()); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_pad_reflect_1d(ctx, a, pad_0, pad_1); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_ROLL -struct test_roll : public test_case { - const int shift0; - const int shift1; - const int shift3; - const int shift4; - - std::string vars() override { - return VARS_TO_STR4(shift0, shift1, shift3, shift4); - } - - test_roll(int shift0 = 3, int shift1 = -2, int shift3 = 1, int shift4 = -1) - : shift0(shift0), shift1(shift1), shift3(shift3), shift4(shift4) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - int64_t ne[4] = {10, 5, 4, 3}; - ggml_tensor * a = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_roll(ctx, a, shift0, shift1, shift3, shift4); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_ARANGE -struct test_arange : public test_case { - const ggml_type type; - const float start; - const float stop; - const float step; - - std::string vars() override { - return VARS_TO_STR4(type, start, stop, step); - } - - test_arange(ggml_type type = GGML_TYPE_F32, - float start = 0.f, float stop = 10.f, float step = 1.f) - : type(type), start(start), stop(stop), step(step) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * out = ggml_arange(ctx, start, stop, step); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_TIMESTEP_EMBEDDING -struct test_timestep_embedding : public test_case { - const ggml_type type; - const std::array ne_a; - const int dim; - const int max_period; - - std::string vars() override { - return VARS_TO_STR4(type, ne_a, dim, max_period); - } - - test_timestep_embedding(ggml_type type = GGML_TYPE_F32, - std::array ne_a = {2, 1, 1, 1}, - int dim = 320, int max_period=10000) - : type(type), ne_a(ne_a), dim(dim), max_period(max_period) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_timestep_embedding(ctx, a, dim, max_period); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_LEAKY_RELU -struct test_leaky_relu : public test_case { - const ggml_type type; - const std::array ne_a; - const float negative_slope; - - std::string vars() override { - return VARS_TO_STR3(type, ne_a, negative_slope); - } - - test_leaky_relu(ggml_type type = GGML_TYPE_F32, - std::array ne_a = {10, 5, 4, 3}, - float negative_slope = 0.1f) - : type(type), ne_a(ne_a), negative_slope(negative_slope) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_leaky_relu(ctx, a, negative_slope, true); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_FLASH_ATTN_EXT -struct test_flash_attn_ext : public test_case { - const int64_t hsk; // K head size - const int64_t hsv; // V head size - const int64_t nh; // num heads - const std::array nr23; // repeat in dim 2 and 3, tests for grouped-query attention - const int64_t kv; // kv size - const int64_t nb; // batch size - - const bool mask; // use mask - const bool sinks; // use sinks - - const float max_bias; // ALiBi - const float logit_softcap; // Gemma 2 - - const ggml_prec prec; - const ggml_type type_K; - const ggml_type type_V; - std::array permute; - - std::string vars() override { - return VARS_TO_STR14(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute); - } - - double max_nmse_err() override { - return 5e-4; - } - - uint64_t op_flops(ggml_tensor * t) override { - GGML_UNUSED(t); - // Just counting matmul costs: - // Q*K^T is nb x hsk x kv, P*V is nb x kv x hsv, per head - return (2 * nh*nr23[0] * nb * (hsk + hsv) * kv)*nr23[1]; - } - - test_flash_attn_ext(int64_t hsk = 128, int64_t hsv = 128, int64_t nh = 32, std::array nr23 = {1, 1}, int64_t kv = 96, int64_t nb = 8, - bool mask = true, bool sinks = false, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32, - ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16, std::array permute = {0, 1, 2, 3}) - : hsk(hsk), hsv(hsv), nh(nh), nr23(nr23), kv(kv), nb(nb), mask(mask), sinks(sinks), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec), - type_K(type_K), type_V(type_V), permute(permute) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - const int64_t hsk_padded = GGML_PAD(hsk, ggml_blck_size(type_K)); - const int64_t hsv_padded = GGML_PAD(hsv, ggml_blck_size(type_V)); - - auto const &create_permuted = [&](ggml_type type, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, bool is_view) -> ggml_tensor * { - int64_t ne[4] = {ne0, ne1, ne2, ne3}; - int64_t ne_perm[4]; - for (int i = 0; i < 4; ++i) { - ne_perm[permute[i]] = ne[i]; - } - ggml_tensor * t; - if (is_view) { - ggml_tensor * t0 = ggml_new_tensor_4d(ctx, type, ne_perm[0], 2*ne_perm[1], ne_perm[2], ne_perm[3]); - t = ggml_view_4d(ctx, t0, ne_perm[0], ne_perm[1], ne_perm[2], ne_perm[3], t0->nb[1], t0->nb[2], t0->nb[3], 0); - } else { - t = ggml_new_tensor_4d(ctx, type, ne_perm[0], ne_perm[1], ne_perm[2], ne_perm[3]); - } - if (permute != std::array{0, 1, 2, 3}) { - t = ggml_permute(ctx, t, permute[0], permute[1], permute[2], permute[3]); - } - return t; - }; - - ggml_tensor * q = create_permuted(GGML_TYPE_F32, hsk_padded, nb, nh*nr23[0], nr23[1], false); - ggml_set_name(q, "q"); - - ggml_tensor * k = create_permuted(type_K, hsk_padded, kv, nh, nr23[1], true); // the K tensor is usually a view of the K cache - ggml_set_name(k, "k"); - - ggml_tensor * v = nullptr; - if (type_K == type_V && hsk_padded == 576 && hsv_padded == 512) { - // TODO: this branch should become a separate test case parameter instead of hardcoding this for these head shapes - - // in this branch, the V cache is sub-view of the K cache. this is used by some MLA-based models - // for more info: - // - https://github.com/ggml-org/llama.cpp/pull/13435 - // - https://github.com/ggml-org/llama.cpp/pull/18953#issuecomment-3774948392 - // - https://github.com/ggml-org/llama.cpp/pull/18986 - v = ggml_view_4d(ctx, k, hsv_padded, kv, nh, nr23[1], k->nb[1], k->nb[2], k->nb[3], 0); - } else { - v = create_permuted(type_V, hsv_padded, kv, nh, nr23[1], true); // the V tensor is usually a view of the V cache - } - ggml_set_name(v, "v"); - - ggml_tensor * m = nullptr; - if (mask) { - m = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, kv, nb, 1, nr23[1]); - ggml_set_name(m, "m"); - } - - ggml_tensor * s = nullptr; - if (sinks) { - s = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, q->ne[2]); - ggml_set_name(s, "s"); - } - - ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf(hsk), max_bias, logit_softcap); - ggml_flash_attn_ext_add_sinks(out, s); - ggml_flash_attn_ext_set_prec (out, prec); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (strcmp(t->name, "s") == 0) { - // make the sink values more noticeable in order to trigger a test failure when the implementation is wrong - init_tensor_uniform(t, -10.0f, 10.0f); - } else if (strcmp(t->name, "m") == 0) { - init_tensor_kq_mask(t); - } else { - init_tensor_uniform(t); - } - } - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_CROSS_ENTROPY_LOSS -struct test_cross_entropy_loss : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_cross_entropy_loss(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * logits = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(logits); - ggml_set_name(logits, "logits"); - - ggml_tensor * labels = ggml_new_tensor(ctx, type, 4, ne.data()); - // The labels are assumed to be constant -> no gradients. - ggml_set_name(labels, "labels"); - - // Ensure labels add up to 1: - labels = ggml_soft_max(ctx, labels); - ggml_set_name(labels, "labels_normalized"); - - ggml_tensor * out = ggml_cross_entropy_loss(ctx, logits, labels); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - // For larger abs. diffs between logits softmax is more linear, therefore more precise num. gradients. - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -100.0f, 100.0f); - } - } - - float grad_eps() override { - return 1.0f; - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_CROSS_ENTROPY_LOSS_BACK -struct test_cross_entropy_loss_back : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_cross_entropy_loss_back(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * grad = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); - ggml_set_name(grad, "grad"); - - ggml_tensor * logits = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(logits, "logits"); - - ggml_tensor * labels = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_name(labels, "labels"); - - // Ensure labels add up to 1: - labels = ggml_soft_max(ctx, labels); - ggml_set_name(labels, "labels_normalized"); - - ggml_tensor * out = ggml_cross_entropy_loss_back(ctx, grad, logits, labels); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_OPT_STEP_ADAMW -struct test_opt_step_adamw : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { - return VARS_TO_STR2(type, ne); - } - - test_opt_step_adamw(ggml_type type = GGML_TYPE_F32, - std::array ne = {10, 5, 4, 3}) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); - ggml_set_param(a); // Despite tensor a having gradients the output tensor will not. - ggml_set_name(a, "a"); - - ggml_tensor * grad = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); - ggml_set_name(grad, "grad"); - - ggml_tensor * grad_m = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); - ggml_set_name(grad_m, "grad_m"); - - ggml_tensor * grad_v = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); - ggml_set_name(grad_v, "grad_v"); - - ggml_tensor * adamw_params = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 7); - ggml_set_name(adamw_params, "adamw_params"); - - ggml_tensor * out = ggml_opt_step_adamw(ctx, a, grad, grad_m, grad_v, adamw_params); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, 0.0f, 1.0f); // grad_v and adamw_params need non-negative values. - } - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_OPT_STEP_SGD -struct test_opt_step_sgd : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { return VARS_TO_STR2(type, ne); } - - test_opt_step_sgd(ggml_type type = GGML_TYPE_F32, - std::array ne = { 10, 5, 4, 3 }) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); - ggml_set_param(a); // Despite tensor a having gradients the output tensor will not. - ggml_set_name(a, "a"); - - ggml_tensor * grad = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); - ggml_set_name(grad, "grad"); - - ggml_tensor * sgd_params = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 2); - ggml_set_name(sgd_params, "sgd_params"); - - ggml_tensor * out = ggml_opt_step_sgd(ctx, a, grad, sgd_params); - - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, 0.0f, 1.0f); // sgd_params need non-negative values. - } - } - - bool grad_precise() override { - return true; - } -}; - -// GGML_OP_CUMSUM -struct test_cumsum : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { return VARS_TO_STR2(type, ne); } - - test_cumsum(ggml_type type = GGML_TYPE_F32, - std::array ne = { 10, 5, 4, 3 }) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_cumsum(ctx, a); - - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -1.0f, 1.0f); - } - } -}; - -// GGML_OP_XIELU -struct test_xielu : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { return VARS_TO_STR2(type, ne); } - - test_xielu(ggml_type type = GGML_TYPE_F32, - std::array ne = { 10, 5, 4, 3 }) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); - ggml_set_param(a); - ggml_set_name(a, "a"); - - float alpha_n = 4.0f; - float alpha_p = 20.0f; - float beta = 0.5f; - float eps = 0.0000001f; - - ggml_tensor * out = ggml_xielu(ctx, a, alpha_n, alpha_p, beta, eps); - - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -1.0f, 1.0f); - } - } -}; - -// GGML_OP_TRI -struct test_tri : public test_case { - const ggml_type type; - const std::array ne; - const ggml_tri_type tri_type; - - std::string vars() override { return VARS_TO_STR3(type, ne, tri_type); } - - test_tri(ggml_tri_type tri_type, ggml_type type = GGML_TYPE_F32, - std::array ne = { 10, 10, 4, 3 }) - : type(type), ne(ne), tri_type(tri_type) { - GGML_ASSERT(ne[0] == ne[1]); - } - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_tri(ctx, a, tri_type); - - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - init_tensor_uniform(t, -1.0f, 1.0f); - } - } -}; - -// GGML_OP_FILL -struct test_fill : public test_case { - const ggml_type type; - const std::array ne; - float c; - - std::string vars() override { return VARS_TO_STR3(type, ne, c); } - - test_fill(float c, ggml_type type = GGML_TYPE_F32, - std::array ne = { 10, 10, 4, 3 }) - : type(type), ne(ne), c(c) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_fill(ctx, a, c); - - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_SOLVE_TRI -struct test_solve_tri : public test_case { - const ggml_type type; - const std::array ne_lhs; - const std::array ne_rhs; - - std::string vars() override { return VARS_TO_STR3(type, ne_lhs, ne_rhs); } - - uint64_t op_flops(ggml_tensor * t) override { - GGML_UNUSED(t); - int64_t n = ne_lhs[0]; - int64_t k = ne_rhs[0]; - int64_t batch = ne_lhs[2] * ne_lhs[3]; - // n * (n + 1) / 2 non-zero elements of lhs, 2 flops each, for each col of rhs - return n * (n + 1) * k * batch; - } - - test_solve_tri(ggml_type type = GGML_TYPE_F32, - std::array ne_lhs = { 10, 10, 4, 3 }, - std::array ne_rhs = { 3, 10, 4, 3 } - ) - : type(type), ne_lhs(ne_lhs), ne_rhs(ne_rhs) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne_lhs[0], ne_lhs[1], ne_lhs[2], ne_lhs[3]); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * b = ggml_new_tensor_4d(ctx, type, ne_rhs[0], ne_rhs[1], ne_rhs[2], ne_rhs[3]); - ggml_set_param(b); - ggml_set_name(b, "b"); - - ggml_tensor * out = ggml_solve_tri(ctx, a, b, true, true, false); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (strcmp(t->name, "a") == 0) { - // note: avoid zeros in the diagonal - init_tensor_tril(t, 0.1, 1.0f); - } else { - init_tensor_uniform(t, -1.0f, 1.0f); - } - } - } -}; - -// GGML_OP_DIAG -struct test_diag : public test_case { - const ggml_type type; - const std::array ne; - - std::string vars() override { return VARS_TO_STR2(type, ne); } - - test_diag(ggml_type type = GGML_TYPE_F32, - std::array ne = { 10, 1, 4, 3 }) - : type(type), ne(ne) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - GGML_ASSERT(ne[1] == 1); - ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); - ggml_set_param(a); - ggml_set_name(a, "a"); - - ggml_tensor * out = ggml_diag(ctx, a); - ggml_set_name(out, "out"); - - return out; - } -}; - -// GGML_OP_LIGHTNING_INDEXER -struct test_lightning_indexer : public test_case { - const int64_t hsk; // indexer K head size - const int64_t nh; // num indexer heads - const int64_t kv; // kv size - const int64_t nb; // batch size - const int64_t ns; // num streams - const int64_t nm; // ne[3] of mask - - const ggml_type type_K; - - std::string vars() override { - return VARS_TO_STR7(hsk, nh, kv, nb, ns, nm, type_K); - } - - double max_nmse_err() override { - return 1e-6; - } - - uint64_t op_flops(ggml_tensor * t) override { - GGML_UNUSED(t); - return ((2 * hsk + 2) * nh + 1) * kv * nb * ns; - } - - test_lightning_indexer(int64_t hsk = 128, int64_t nh = 64, int64_t kv = 256, int64_t nb = 128, int64_t ns = 1, int64_t nm = 1, ggml_type type_K = GGML_TYPE_F16) - : hsk(hsk), nh(nh), kv(kv), nb(nb), ns(ns), nm(nm), type_K(type_K) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, hsk, nh, nb, ns); - ggml_set_param(q); - ggml_set_name(q, "q"); - - ggml_tensor * k = ggml_new_tensor_4d(ctx, type_K, hsk, 1, kv, ns); - ggml_set_param(k); - ggml_set_name(k, "k"); - - ggml_tensor * w = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, nh, nb, 1, ns); - ggml_set_param(w); - ggml_set_name(w, "w"); - - ggml_tensor * m = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, kv, nb, 1, nm); - ggml_set_param(m); - ggml_set_name(m, "m"); - - ggml_tensor * out = ggml_lightning_indexer(ctx, q, k, w, m); - ggml_set_name(out, "out"); - - return out; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (strcmp(t->name, "m") == 0) { - init_tensor_kq_mask(t); - } else { - init_tensor_uniform(t); - } - } - } -}; - -// Deserializable generic test case -struct input_tensor { - ggml_type type; - std::array ne; - std::array nb; // strides (0 = use default contiguous strides) -}; - -static bool is_non_contiguous(const input_tensor & src) { - if (src.nb[0] == 0) { - return false; - } - const size_t default_nb0 = ggml_type_size(src.type); - const size_t default_nb1 = default_nb0 * (src.ne[0] / ggml_blck_size(src.type)); - const size_t default_nb2 = default_nb1 * src.ne[1]; - const size_t default_nb3 = default_nb2 * src.ne[2]; - return src.nb[0] != default_nb0 || - src.nb[1] != default_nb1 || - src.nb[2] != default_nb2 || - src.nb[3] != default_nb3; -} - -static std::string var_to_str(const std::vector& sources) { - std::ostringstream oss; - bool first = true; - for (const auto& src : sources) { - if (!first) oss << ","; - oss << ggml_type_name(src.type) << "[" << src.ne[0] << "," << src.ne[1] << "," << src.ne[2] << "," << src.ne[3] << "]"; - if (is_non_contiguous(src)) { - oss << "nb[" << src.nb[0] << "," << src.nb[1] << "," << src.nb[2] << "," << src.nb[3] << "]"; - } - first = false; - } - return oss.str(); -} - -static std::string var_to_str(const std::array& params) { - std::ostringstream oss; - oss << "["; - bool first = true; - for (size_t i = 0; i < params.size(); ++i) { - if (params[i] != 0) { - if (!first) oss << ","; - oss << i << ":" << params[i]; - first = false; - } - } - oss << "]"; - return oss.str(); -} - - -struct test_generic_op : public test_case { - const ggml_op op; - const ggml_type type; - const std::array ne; - const std::array op_params; - - const std::vector sources; - const std::string name; - - std::string vars() override { - if (name.empty()) { - return VARS_TO_STR4(type, ne, op_params, sources); - } - - return VARS_TO_STR5(name, type, ne, op_params, sources); - } - - test_generic_op(ggml_op op, ggml_type type, std::array ne, - std::array op_params, - std::vector sources, std::string name = "") - : op(op), type(type), ne(ne), op_params(op_params), sources(sources), name(std::move(name)) {} - - ggml_tensor * build_graph(ggml_context * ctx) override { - const size_t source_count = std::min(sources.size(), (size_t)GGML_MAX_SRC); - - std::array source_tensors; - for (size_t i = 0; i < source_count; ++i) { - const input_tensor& src = sources[i]; - - if (is_non_contiguous(src)) { - size_t total_size; - const size_t blck_size = ggml_blck_size(src.type); - if (blck_size == 1) { - total_size = ggml_type_size(src.type); - for (int d = 0; d < 4; d++) { - total_size += (src.ne[d] - 1) * src.nb[d]; - } - } else { - total_size = src.ne[0] * src.nb[0] / blck_size; - for (int d = 1; d < 4; d++) { - total_size += (src.ne[d] - 1) * src.nb[d]; - } - } - - // Convert bytes to elements, padded to block size for quantized types - const size_t type_size = ggml_type_size(src.type); - size_t backing_elements = (total_size * blck_size + type_size - 1) / type_size; - backing_elements = ((backing_elements + blck_size - 1) / blck_size) * blck_size; - ggml_tensor * backing = ggml_new_tensor_1d(ctx, src.type, backing_elements); - source_tensors[i] = ggml_view_4d(ctx, backing, - src.ne[0], src.ne[1], src.ne[2], src.ne[3], - src.nb[1], src.nb[2], src.nb[3], 0); - // nb[0] does not get set by view_4d, so set it manually - source_tensors[i]->nb[0] = src.nb[0]; - } else { - source_tensors[i] = ggml_new_tensor_4d(ctx, src.type, src.ne[0], src.ne[1], src.ne[2], src.ne[3]); - } - } - - // Ops with an inplace flag create a view of src[0] as their output. - bool inplace = false; - if (op == GGML_OP_SET || op == GGML_OP_ACC) { - inplace = op_params[4] != 0; - } else if (op == GGML_OP_ADD_REL_POS) { - inplace = op_params[0] != 0; - } - - ggml_tensor * out; - if (inplace && source_count > 0) { - out = ggml_view_tensor(ctx, source_tensors[0]); - } else { - out = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); - } - out->op = op; - for (size_t i = 0; i < source_count; ++i) { - out->src[i] = source_tensors[i]; - } - - memcpy(out->op_params, op_params.data(), GGML_MAX_OP_PARAMS); - ggml_set_name(out, "out"); - - return out; - } - - double max_nmse_err() override { - switch (op) { - case GGML_OP_MUL_MAT: - case GGML_OP_MUL_MAT_ID: - case GGML_OP_OUT_PROD: - case GGML_OP_CONV_TRANSPOSE_2D: - case GGML_OP_IM2COL: - case GGML_OP_CONV_2D: - case GGML_OP_CONV_3D: - case GGML_OP_SET_ROWS: - case GGML_OP_CPY: - return 5e-4; - case GGML_OP_SOFT_MAX: - return 1e-6; - case GGML_OP_RWKV_WKV7: - return 5e-3; - case GGML_OP_FLASH_ATTN_EXT: - { - // Scale error with kv length to account for accumulating floating point error - const int64_t kv = sources[1].ne[1]; - return 5e-4 * std::max(1.0, kv / 20000.0); - } - default: - return 1e-7; - } - } - - void initialize_tensors(ggml_context * ctx) override { - ggml_tensor * out = ggml_get_tensor(ctx, "out"); - - std::random_device rd; - std::default_random_engine rng(rd()); - - for (size_t i = 0; i < sources.size() && i < GGML_MAX_SRC; i++) { - ggml_tensor * t = out->src[i]; - if (!t) { - break; - } - - // FLASH_ATTN_EXT: src[3] is the KQ mask - if (op == GGML_OP_FLASH_ATTN_EXT && i == 3) { - init_tensor_kq_mask(t); - continue; - } - - if (t->type == GGML_TYPE_I32 || t->type == GGML_TYPE_I64) { - if (op == GGML_OP_GET_ROWS || op == GGML_OP_GET_ROWS_BACK) { - const int64_t num_rows = sources[0].ne[1]; - const int64_t nels = ggml_nelements(t); - std::vector data(nels); - std::uniform_int_distribution dist(0, num_rows - 1); - for (int64_t i = 0; i < nels; i++) { - data[i] = dist(rng); - } - ggml_backend_tensor_set(t, data.data(), 0, nels * sizeof(int32_t)); - } else if (op == GGML_OP_SET_ROWS) { - init_set_rows_row_ids(t, ne[1]); - } else if (op == GGML_OP_ROPE) { - const int mode = op_params[2]; - const int64_t nels = (mode & GGML_ROPE_TYPE_MROPE) ? ne[2] * 4 : ne[2]; - std::vector data(nels); - std::uniform_int_distribution dist(0, ne[2] - 1); - for (int64_t i = 0; i < nels; i++) { - data[i] = dist(rng); - } - ggml_backend_tensor_set(t, data.data(), 0, nels * sizeof(int32_t)); - } else if (op == GGML_OP_MUL_MAT_ID || op == GGML_OP_ADD_ID) { - const int64_t n_expert = (op == GGML_OP_MUL_MAT_ID) ? sources[0].ne[2] : sources[1].ne[1]; - for (int64_t r = 0; r < ggml_nrows(t); r++) { - std::vector data(t->ne[0]); - for (int32_t i = 0; i < t->ne[0]; i++) { - data[i] = i % n_expert; - } - std::shuffle(data.begin(), data.end(), rng); - ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); - } - } else if (op == GGML_OP_SSM_SCAN) { - for (int64_t r = 0; r < ggml_nrows(t); r++) { - std::vector data(t->ne[0]); - for (int32_t i = 0; i < t->ne[0]; i++) { - data[i] = i; - } - std::shuffle(data.begin(), data.end(), rng); - ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); - } - } else { - init_tensor_uniform(t); - } - } else { - init_tensor_uniform(t); - } - } - } -}; - - -enum llm_norm_type { - LLM_NORM, - LLM_NORM_RMS, -}; - -struct llama_hparams { - uint32_t n_vocab; - uint32_t n_embd; - uint32_t n_head; - uint32_t n_head_kv; - static constexpr uint32_t n_layer = 1; - uint32_t n_rot; - uint32_t n_embd_head; // dimension of values (d_v) - uint32_t n_ff; - - float f_norm_eps; - float f_norm_rms_eps; - - // cparams - static constexpr uint32_t n_ctx = 512; // user-specified context size - static constexpr uint32_t n_ctx_orig = n_ctx; - - // batch - int32_t n_tokens; - - // llm_build_context - static constexpr int32_t n_kv = 32; // size of KV cache to consider (n_kv <= n_ctx - static constexpr int32_t kv_head = 1; // index of where we store new KV data in the cache - - uint32_t n_embd_gqa() const { // dimension of key embeddings across all k-v heads - return n_embd_head * n_head_kv; - } -}; - -// LLM base class -struct test_llm : public test_case { - llama_hparams hp; - -protected: - test_llm(llama_hparams hp) - : hp(std::move(hp)) { - } - -public: - struct ggml_tensor * llm_build_norm( - struct ggml_context * ctx, - struct ggml_tensor * cur, - struct ggml_tensor * mw, - struct ggml_tensor * mb, - llm_norm_type type) { - switch (type) { - case LLM_NORM: cur = ggml_norm (ctx, cur, hp.f_norm_eps); break; - case LLM_NORM_RMS: cur = ggml_rms_norm(ctx, cur, hp.f_norm_rms_eps); break; - } - cur = ggml_mul(ctx, cur, mw); - if (mb) { - cur = ggml_add(ctx, cur, mb); - } - return cur; - } - - void llm_build_kv_store( - struct ggml_context * ctx, - struct ggml_tensor * k_l, - struct ggml_tensor * v_l, - struct ggml_tensor * k_cur, - struct ggml_tensor * v_cur) { - // compute the transposed [n_tokens, n_embd] V matrix - struct ggml_tensor * v_cur_t = ggml_transpose(ctx, ggml_reshape_2d(ctx, v_cur, hp.n_embd_gqa(), hp.n_tokens)); - - struct ggml_tensor * k_cache_view = ggml_view_1d(ctx, k_l, hp.n_tokens*hp.n_embd_gqa(), - (ggml_row_size(k_l->type, hp.n_embd_gqa()))*hp.kv_head); - - struct ggml_tensor * v_cache_view = ggml_view_2d(ctx, v_l, hp.n_tokens, hp.n_embd_gqa(), - ( hp.n_ctx)*ggml_element_size(v_l), - (hp.kv_head)*ggml_element_size(v_l)); - - // important: storing RoPE-ed version of K in the KV cache! - ggml_cpy(ctx, k_cur, k_cache_view); - ggml_cpy(ctx, v_cur_t, v_cache_view); - } - - struct ggml_tensor * llm_build_kqv( - struct ggml_context * ctx, - struct ggml_tensor * k_l, - struct ggml_tensor * v_l, - struct ggml_tensor * q_cur, - struct ggml_tensor * kq_mask, - float kq_scale) { - struct ggml_tensor * q = ggml_permute(ctx, q_cur, 0, 2, 1, 3); - - struct ggml_tensor * k = - ggml_view_3d(ctx, k_l, - hp.n_embd_head, hp.n_kv, hp.n_head_kv, - ggml_row_size(k_l->type, hp.n_embd_gqa()), - ggml_row_size(k_l->type, hp.n_embd_head), - 0); - - struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q); - - kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_scale, 0.0f); - - // split cached v into n_head heads - struct ggml_tensor * v = - ggml_view_3d(ctx, v_l, - hp.n_kv, hp.n_embd_head, hp.n_head_kv, - ggml_element_size(v_l)*hp.n_ctx, - ggml_element_size(v_l)*hp.n_ctx*hp.n_embd_head, - 0); - - struct ggml_tensor * kqv = ggml_mul_mat(ctx, v, kq); - - struct ggml_tensor * kqv_merged = ggml_permute(ctx, kqv, 0, 2, 1, 3); - - struct ggml_tensor * cur = ggml_cont_2d(ctx, kqv_merged, hp.n_embd_head*hp.n_head, hp.n_tokens); - - struct ggml_tensor * wo = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_embd); - cur = ggml_mul_mat(ctx, wo, cur); - - return cur; - } - - void initialize_tensors(ggml_context * ctx) override { - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I32) { - // pos - std::vector data(hp.n_tokens); - for (int i = 0; i < hp.n_tokens; i++) { - data[i] = rand() % hp.n_ctx; - } - ggml_backend_tensor_set(t, data.data(), 0, hp.n_tokens * sizeof(int)); - } else { - init_tensor_uniform(t); - } - } - } -}; - -// Llama -struct test_llama : public test_llm { - static constexpr float freq_base = 10000.0f; - static constexpr float freq_scale = 1.0f; - static constexpr float ext_factor = 0.0f; - static constexpr float attn_factor = 1.0f; - static constexpr float beta_fast = 32.0f; - static constexpr float beta_slow = 1.0f; - bool fused; - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "LLAMA"; - } - - std::string vars() override { - auto n_tokens = hp.n_tokens; - return VARS_TO_STR1(n_tokens); - } - - double max_nmse_err() override { - return 2e-3; - } - - bool run_whole_graph() override { return fused; } - - test_llama(int n_tokens = 1, bool fused = false) - : test_llm({ - /*n_vocab =*/ 32000, - /*n_embd =*/ 3200, - /*n_head =*/ 32, - /*n_head_kv =*/ 32, - /*n_rot =*/ 100, - /*n_embd_head =*/ 100, - /*n_ff =*/ 8640, - /*f_norm_eps =*/ 0.f, - /*f_norm_rms_eps =*/ 1e-5f, - /*n_tokens =*/ n_tokens, - }) - , fused(fused) - { - } - - ggml_tensor * build_graph(ggml_context * ctx) override { - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hp.n_embd, hp.n_tokens); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, hp.n_tokens); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx, GGML_TYPE_F16, hp.n_kv, hp.n_tokens, 1); - - ggml_tensor * k_l = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, 1638400); - ggml_tensor * v_l = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, 1638400); - - for (uint32_t il = 0; il < hp.n_layer; ++il) { - struct ggml_tensor * inpSA = inpL; - - // norm - ggml_tensor * attn_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); - cur = llm_build_norm(ctx, inpL, attn_norm, nullptr, LLM_NORM_RMS); - - // self-attention - { - ggml_tensor * wq = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_embd); - ggml_tensor * wk = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_embd_gqa()); - ggml_tensor * wv = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_embd_gqa()); - - // compute Q and K and RoPE them - struct ggml_tensor * Qcur = ggml_mul_mat(ctx, wq, cur); - struct ggml_tensor * Kcur = ggml_mul_mat(ctx, wk, cur); - struct ggml_tensor * Vcur = ggml_mul_mat(ctx, wv, cur); - - Qcur = ggml_rope_ext( - ctx, ggml_reshape_3d(ctx, Qcur, hp.n_embd_head, hp.n_head, hp.n_tokens), inp_pos, nullptr, - hp.n_rot, 0, hp.n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx, ggml_reshape_3d(ctx, Kcur, hp.n_embd_head, hp.n_head_kv, hp.n_tokens), inp_pos, nullptr, - hp.n_rot, 0, hp.n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - llm_build_kv_store(ctx, k_l, v_l, Kcur, Vcur); - - cur = llm_build_kqv(ctx, k_l, v_l, Qcur, KQ_mask, 1.0f/sqrtf(float(hp.n_embd_head))); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx, cur, inpSA); - - // feed-forward network - ggml_tensor * ffn_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); - cur = llm_build_norm(ctx, ffn_inp, ffn_norm, nullptr, LLM_NORM_RMS); - - ggml_tensor * ffn_gate = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_ff); - ggml_tensor * ffn_down = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_ff, hp.n_embd); - ggml_tensor * ffn_up = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_ff); - struct ggml_tensor * tmp = ggml_mul_mat(ctx, ffn_up, cur); - cur = ggml_mul_mat(ctx, ffn_gate, cur); - cur = ggml_silu(ctx, cur); - cur = ggml_mul(ctx, cur, tmp); - cur = ggml_mul_mat(ctx, ffn_down, cur); - - cur = ggml_add(ctx, cur, ffn_inp); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - ggml_tensor * output_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); - cur = llm_build_norm(ctx, cur, output_norm, nullptr, LLM_NORM_RMS); - - // lm_head - ggml_tensor * output = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_vocab); - cur = ggml_mul_mat(ctx, output, cur); - - return cur; - } -}; - -// Falcon -struct test_falcon : public test_llm { - static constexpr float freq_base = 10000.0f; - static constexpr float freq_scale = 1.0f; - static constexpr float ext_factor = 0.0f; - static constexpr float attn_factor = 1.0f; - static constexpr float beta_fast = 32.0f; - static constexpr float beta_slow = 1.0f; - - std::string op_desc(ggml_tensor * t) override { - GGML_UNUSED(t); - return "FALCON"; - } - - std::string vars() override { - auto n_tokens = hp.n_tokens; - return VARS_TO_STR1(n_tokens); - } - - double max_nmse_err() override { - return 2e-3; - } - - test_falcon(int n_tokens = 1) - : test_llm({ - /*n_vocab =*/ 32000, - /*n_embd =*/ 3200, - /*n_head =*/ 50, - /*n_head_kv =*/ 1, - /*n_rot =*/ 64, - /*n_embd_head =*/ 64, - /*n_ff =*/ 8640, - /*f_norm_eps =*/ 1e-5f, - /*f_norm_rms_eps =*/ 0.f, - /*n_tokens =*/ n_tokens, - }) { - } - - ggml_tensor * build_graph(ggml_context * ctx) override { - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hp.n_embd, hp.n_tokens); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, hp.n_tokens); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx, GGML_TYPE_F16, hp.n_kv, hp.n_tokens, 1); - - ggml_tensor * k_l = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, 1638400); - ggml_tensor * v_l = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, 1638400); - - for (uint32_t il = 0; il < hp.n_layer; ++il) { - // norm - ggml_tensor * attn_norm_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); - ggml_tensor * attn_norm_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); - ggml_tensor * attn_norm = llm_build_norm(ctx, inpL, attn_norm_w, attn_norm_b, LLM_NORM); - - // self-attention - { - cur = attn_norm; - - ggml_tensor * wqkv = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_embd + 2*hp.n_embd_gqa()); - - cur = ggml_mul_mat(ctx, wqkv, cur); - - struct ggml_tensor * Qcur = ggml_cont(ctx, ggml_view_2d(ctx, cur, hp.n_embd, hp.n_tokens, cur->nb[1], 0*sizeof(float)*(hp.n_embd))); - struct ggml_tensor * Kcur = ggml_cont(ctx, ggml_view_2d(ctx, cur, hp.n_embd_gqa(), hp.n_tokens, cur->nb[1], 1*sizeof(float)*(hp.n_embd))); - struct ggml_tensor * Vcur = ggml_cont(ctx, ggml_view_2d(ctx, cur, hp.n_embd_gqa(), hp.n_tokens, cur->nb[1], 1*sizeof(float)*(hp.n_embd + hp.n_embd_gqa()))); - - Qcur = ggml_reshape_3d(ctx, Qcur, hp.n_embd_head, hp.n_head, hp.n_tokens); - Kcur = ggml_reshape_3d(ctx, Kcur, hp.n_embd_head, hp.n_head_kv, hp.n_tokens); - - // using mode = 2 for neox mode - Qcur = ggml_rope_ext( - ctx, Qcur, inp_pos, nullptr, hp.n_rot, 2, hp.n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx, Kcur, inp_pos, nullptr, hp.n_rot, 2, hp.n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - - llm_build_kv_store(ctx, k_l, v_l, Kcur, Vcur); - - cur = llm_build_kqv(ctx, k_l, v_l, Qcur, KQ_mask, 1.0f/sqrtf(float(hp.n_embd_head))); - } - - struct ggml_tensor * ffn_inp = cur; - - // feed forward - { - ggml_tensor * ffn_up = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_ff); - ggml_tensor * ffn_down = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_ff, hp.n_embd); - cur = attn_norm; - cur = ggml_mul_mat(ctx, ffn_up, cur); - cur = ggml_gelu(ctx, cur); - cur = ggml_mul_mat(ctx, ffn_down, cur); - } - - cur = ggml_add(ctx, cur, ffn_inp); - - cur = ggml_add(ctx, cur, inpL); - - // input for next layer - inpL = cur; - } - - cur = inpL; - - ggml_tensor * output_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); - ggml_tensor * output_norm_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); - cur = llm_build_norm(ctx, cur, output_norm, output_norm_b, LLM_NORM); - - // lm_head - ggml_tensor * output = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, hp.n_embd, hp.n_vocab); - cur = ggml_mul_mat(ctx, output, cur); - - return cur; - } -}; - - -// ########################################### -// ## Section 3: GGML Op Test Instantiation ## -// ########################################### -static const ggml_type all_types[] = { - GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, - GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, - GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, - GGML_TYPE_Q8_0, - GGML_TYPE_Q1_0, - GGML_TYPE_MXFP4, GGML_TYPE_NVFP4, - GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, - GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, - GGML_TYPE_Q6_K, - // GGML_TYPE_TQ1_0, GGML_TYPE_TQ2_0, // TODO: implement for all backends - GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, - GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, - GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS, -}; - -static const ggml_type base_types[] = { - GGML_TYPE_F32, GGML_TYPE_F16, - GGML_TYPE_Q8_0, // for I8MM tests - GGML_TYPE_Q1_0, - GGML_TYPE_Q4_0, - GGML_TYPE_Q4_1, // for I8MM tests - GGML_TYPE_Q4_K, - GGML_TYPE_MXFP4, GGML_TYPE_NVFP4, // TODO: or "other" - GGML_TYPE_IQ2_XXS -}; - -static const ggml_type other_types[] = { - GGML_TYPE_Q4_1, - GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, - GGML_TYPE_Q8_0, - GGML_TYPE_Q1_0, - GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, - GGML_TYPE_Q5_K, - GGML_TYPE_Q6_K, - // GGML_TYPE_TQ1_0, GGML_TYPE_TQ2_0, // TODO: implement for all backends - GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, - GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, - GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS, - GGML_TYPE_BF16, -}; - -#ifdef _MSC_VER -// Workaround long compile time with msvc -#pragma optimize("", off) -#endif - -// Test cases for evaluation: should try to cover edge cases while using small input sizes to keep the runtime low -static std::vector> make_test_cases_eval() { - std::vector> test_cases; - std::default_random_engine rng(0); - - // unary ops - for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { - for (int v : {0, 1}) { - for (int op = 0; op < GGML_UNARY_OP_COUNT; op++) { - if (op == GGML_UNARY_OP_XIELU) { - continue; // need extra params, separate test - } - test_cases.emplace_back(new test_unary((ggml_unary_op) op, type, { 128, 2, 2, 2 }, v)); - test_cases.emplace_back(new test_unary((ggml_unary_op) op, type, { 5, 7, 11, 13 }, v)); - } - } - } - - // fused relu + sqr (squared ReLU) - for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { - test_cases.emplace_back(new test_relu_sqr(type, { 128, 2, 2, 2 })); - test_cases.emplace_back(new test_relu_sqr(type, { 5, 7, 11, 13 })); - } - - // SNAKE activation fusion: x + sin(a*x)^2 * inv_b - for (ggml_type type : { GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16 }) { - test_cases.emplace_back(new test_snake_fuse(type, { 5, 7, 1, 1})); // primes sub-block - test_cases.emplace_back(new test_snake_fuse(type, { 33, 32, 1, 1})); // boundary - test_cases.emplace_back(new test_snake_fuse(type, {1025, 13, 1, 1})); // large prime, grid-stride - test_cases.emplace_back(new test_snake_fuse(type, { 128, 16, 1, 1})); // power-of-two - test_cases.emplace_back(new test_snake_fuse(type, { 256, 192, 1, 1})); // BigVGAN-ish - // higher-rank shapes: matcher must reject fusion, fallback to naive chain - test_cases.emplace_back(new test_snake_fuse(type, { 64, 32, 2, 1})); // ne[2] > 1 - test_cases.emplace_back(new test_snake_fuse(type, { 64, 32, 1, 2})); // ne[3] > 1 - test_cases.emplace_back(new test_snake_fuse(type, { 64, 32, 2, 3})); // ne[2] > 1 and ne[3] > 1 - } - - // glu ops - for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { - for (int v : {0, 1}) { - for (int op = 0; op < GGML_GLU_OP_COUNT; op++) { - if (op == GGML_GLU_OP_SWIGLU_OAI) { - // SWIGLU_OAI is handled separately - continue; - } - - for (bool swapped : {false, true}) { - test_cases.emplace_back(new test_glu((ggml_glu_op) op, type, { 128, 2, 2, 2 }, v, swapped)); - test_cases.emplace_back(new test_glu((ggml_glu_op) op, type, { 5, 7, 11, 13 }, v, swapped)); - } - - test_cases.emplace_back(new test_glu_split((ggml_glu_op) op, type, { 128, 2, 2, 2 }, v)); - test_cases.emplace_back(new test_glu_split((ggml_glu_op) op, type, { 5, 7, 11, 13 }, v)); - } - } - } - - for (int v : {0, 1}) { - for (float alpha : {.5f, 1.702f}) { - for (float limit : {2.0f, 7.0f}) { - test_cases.emplace_back(new test_swiglu_oai(GGML_TYPE_F32, { 128, 2, 2, 2 }, v, alpha, limit)); - } - } - } - - for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_Q4_0}) { - test_cases.emplace_back(new test_get_rows(type, 300*256, 5, 4, 1, 2, false)); - test_cases.emplace_back(new test_get_rows(type, 256, 80000, 70000, 2, 1, false)); - test_cases.emplace_back(new test_get_rows(type, 256, 5, 4, 700, 100, false)); - } - - test_cases.emplace_back(new test_get_rows(GGML_TYPE_F32, 1, 8, 2, 1, 1, false)); - for (ggml_type type : all_types) { - for (int b : {1, 7}) { - for (bool v : {false, true}) { - test_cases.emplace_back(new test_get_rows(type, 256, 5, 4, b, 1, v)); - } - } - } - for (int b : {1, 7}) { - for (bool v : {false, true}) { - test_cases.emplace_back(new test_get_rows(GGML_TYPE_I32, 256, 5, 4, b, 1, v)); - } - } - - test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 8, 2, 1, false)); - test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 70000, 4, 1, false)); // row count > CUDA grid-y limit (65535) - for (ggml_type type : all_types) { - for (bool v : {false, true}) { - test_cases.emplace_back(new test_get_rows_back(type, 256, 5, 4, 1, v)); - } - } - for (bool v : {false, true}) { - test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_I32, 256, 5, 4, 1, v)); - } - - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_Q8_0, GGML_TYPE_I32, { 256, 5, 1, 3 }, { 1, 1, }, 1, false)); - for (ggml_type type : all_types) { - for (int b : {1, 7}) { - for (bool v : {false, true}) { - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 256, 5, b, 3 }, { 1, 1, }, 1, v)); - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 256, 11, 1, b }, { 2, 3, }, 7, v)); - - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 3*ggml_blck_size(type), 3, b, 1 }, { 2, 3, }, 2, v)); - - if (ggml_blck_size(type) == 1) { - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 31, 3, b, 1 }, { 2, 3, }, 2, v)); - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 33, 5, 1, b }, { 2, 3, }, 1, v)); - } - } - } - } - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, true)); - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, true)); - - for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_VISION }) { - for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { - for (int ne2 : {1, 8, 512}) { - test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, { 128, 32, ne2, 1 }, mode)); - test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, { 128, 32, ne2, 3 }, mode)); - } - } - } - - for (ggml_type type_input : {GGML_TYPE_F32}) { - for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) { - for (int k0 : {1, 3}) { - for (int k1 : {1, 3}) { - for (int s0 : {1, 2}) { - for (int s1 : {1, 2}) { - for (int p0 : {0, 1}) { - for (int p1 : {0, 1}) { - test_cases.emplace_back(new test_pool2d(pool_type, type_input, {10, 10, 3, 1}, k0, k1, s0, s1, p0, p1)); - } - } - } - } - } - } - } - } - - for (ggml_type type_input : {GGML_TYPE_F32}) { - for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) { - for (int k0 : {1, 3}) { - for (int s0 : {1, 2}) { - for (int p0 : {0, 1}) { - test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 10, 3, 2, 1 }, k0, s0, p0)); - test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 11, 1, 3, 2 }, k0, s0, p0)); - test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 128, 2, 1, 3 }, k0, s0, p0)); - } - } - } - } - } - -#if 0 - // >4GB im2col destination. Too slow to run by default. - // Test cases taken from Wan2.1 T2V 1.3B. - test_cases.emplace_back(new test_im2col (GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {832, 480, 192, 4}, {3, 3, 192, 96}, 1, 1, 1, 1, 1, 1, true)); - test_cases.emplace_back(new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {834, 482, 6, 96}, {3, 3,3, 9216}, 96, 1, 1, 1, 0, 0, 0, 1, 1, 1, false)); -#endif - - // im2col 1D - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {3000, 384, 1, 1}, {3, 384, 384, 1}, 1, 0, 1, 0, 1, 0, false)); - for (int s0 : {1, 3}) { - for (int p0 : {0, 3}) { - for (int d0 : {1, 3}) { - test_cases.emplace_back(new test_im2col( - GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {20, 2, 2, 1}, {3, 2, 2, 1}, - s0, 0, p0, 0, d0, 0, false)); - } - } - } - - // im2col 2D - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16)); - for (int s0 : {1, 3}) { - for (int s1 : {1, 3}) { - for (int p0 : {0, 3}) { - for (int p1 : {0, 3}) { - for (int d0 : {1, 3}) { - for (int d1 : {1, 3}) { - test_cases.emplace_back(new test_im2col( - GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {20, 20, 2, 2}, {3, 3, 2, 2}, - s0, s1, p0, p1, d0, d1, true)); - } - } - } - } - } - } - - // extra tests for im2col 2D - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 1, 32}, {3, 3, 1, 32}, 1, 1, 1, 1, 1, 1, true)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 2, 32}, {3, 3, 2, 32}, 1, 1, 1, 1, 1, 1, true)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 1, 1024}, {3, 3, 1, 1024}, 1, 1, 1, 1, 1, 1, true)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 2, 1024}, {3, 3, 2, 1024}, 1, 1, 1, 1, 1, 1, true)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 1, 2048}, {3, 3, 1, 2048}, 1, 1, 1, 1, 1, 1, true)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 2, 2048}, {3, 3, 2, 2048}, 1, 1, 1, 1, 1, 1, true)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 1, 2560}, {3, 3, 1, 2560}, 1, 1, 1, 1, 1, 1, true)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 2, 2560}, {3, 3, 2, 2560}, 1, 1, 1, 1, 1, 1, true)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {5, 5, 1, 32}, {3, 4, 1, 32}, 1, 1, 0, 0, 1, 1, true)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {2, 2, 1536, 729}, {2, 2, 1536, 4096}, 1, 1, 0, 0, 1, 1, true)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {128, 128, 1, 2}, {32, 33, 1, 2}, 1, 1, 1, 1, 1, 1, true)); - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {128, 128, 2, 1}, {33, 34, 2, 1}, 1, 1, 1, 1, 1, 1, true)); - - // im2col 3D - test_cases.emplace_back(new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32)); - test_cases.emplace_back(new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32)); - test_cases.emplace_back(new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16)); - for (int s0 : {1, 3}) { - for (int s1 : {1, 3}) { - for (int s2 : {1, 3}) { - for (int p0 : {0, 3}) { - for (int p1 : {0, 3}) { - for (int p2 : {0, 3}) { - for (int d0 : {1, 3}) { - for (int d1 : {1, 3}) { - for (int d2 : {1, 3}) { - for (int IC : {1, 3}) { - for (bool v : {false, true}) { - test_cases.emplace_back(new test_im2col_3d( - GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {20, 20, 10, 3}, {3, 3, 3, 3}, - IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, v)); - } - } - } - } - } - } - } - } - } - } - } - -// Conv_2D test cases -#ifdef DETAILED_TESTS - // Probably we do not have enough time to execute these in the pipeline. - uint32_t iwh_idx = 0; - uint32_t kwh_idx = 1; - uint32_t Cout_idx = 2; - uint32_t Cin_idx = 3; - uint32_t B_idx = 4; - - std::vector> cases = { - //{IWH, KWH, Cout, Cin, B} - // K=CRS=NPQ=4096 conv_2d matmul performance - {19, 4, 4096, 256, 16}, - // K=128, CRS=128, NPQ=4096 - { 19, 4, 128, 8, 16}, - // K=130, CRS=128, NPQ=4096 - { 19, 4, 130, 8, 16}, - // Edge case: K x CRS is small - { 19, 2, 4, 4, 16}, - // A ConvNet's first layer - { 224, 3, 8, 3, 1 }, - // A ConvNet's first layer with 2x2 convolution, and 1 channel - { 224, 2, 8, 1, 1 }, - // A ConvNet's first layer with 2x2 convolution, and 1 channel, several images in the batch - { 224, 2, 8, 1, 8 }, - // A middle layer of a ConvNet - { 58, 3, 64, 32, 1 }, - // A middle layer of a ConvNet, several images in the batch - { 58, 3, 64, 32, 8 }, - // A deep layer of a ConvNet, several images in the batch - { 16, 3, 256, 128, 8 } - }; - - for (auto kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - for (auto act_case : cases) { - test_cases.emplace_back(new test_conv_2d( - { act_case[iwh_idx], act_case[iwh_idx], act_case[Cin_idx], act_case[B_idx] }, - { act_case[kwh_idx], act_case[kwh_idx], act_case[Cin_idx], act_case[Cout_idx] }, - kernel_type, 1, 1, 0, 0, 1, 1, false)); - } - } -#endif - - // CONV_2D: - auto calc_conv_output_size = [](int64_t ins, int64_t ks, int s, int p, int d) -> int64_t { - return (ins + 2 * p - d * (ks - 1) - 1) / s + 1; - }; - - //uint32_t s0 = 3; - uint32_t s1 = 5; - uint32_t p0 = 5; - //uint32_t p1 = 2; - uint32_t d0 = 2; - uint32_t d1 = 4; - - for (uint32_t s0 : { 1, 3 }) { - for (uint32_t p1 : { 2, 5 }) { - for (uint32_t Cin : { 1, 25 }) { - for (uint32_t Cout : { 1, 12 }) { - for (uint32_t KH : { 1, 2, 3, 11 }) { - for (uint32_t KW : { 1, 2, 3, 11 }) { - for (uint32_t H : { 1, 133 }) { - for (uint32_t W : { 1, 141 }) { - if (calc_conv_output_size(W, KW, s0, p0, d0) > 0 && - calc_conv_output_size(H, KH, s1, p1, d1) > 0) { - for (auto kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - test_cases.emplace_back(new test_conv_2d( - { W, H, Cin, 2 }, { KW, KH, Cin, Cout }, kernel_type, s0, s1, p0, p1, d0, d1, false)); - } - } - } - } - } - } - } - } - } - } - for (auto kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - test_cases.emplace_back(new test_conv_2d({ 256, 256, 192, 1 }, { 3, 3, 192, 96 }, kernel_type, 1, 1, 1, 1, 1, 1, false)); - } - - // sycl backend will limit task global_range < MAX_INT - // test cases for 2D im2col with large input W and H (occurs in stable-diffusion) - // however these cases need to alloc more memory which may fail in some devices (Intel Arc770, etc.) - // these cases are verified (pass) in Intel(R) Data Center GPU Max 1100 (sycl backend) and NV A30 (cuda backend) - // test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {1024, 1024, 256, 1}, {3, 3, 256, 1}, 1, 1, 1, 1, 1, 1, true)); - // test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {1024, 1024, 256, 1}, {3, 3, 256, 1}, 1, 1, 1, 1, 1, 1, true)); - - test_cases.emplace_back(new test_conv_2d_dw({17, 34, 9, 1}, {3, 3, 1, 9}, GGML_TYPE_F32, 1, 0, 1, false)); - test_cases.emplace_back(new test_conv_2d_dw({17, 34, 9, 1}, {3, 3, 1, 9}, GGML_TYPE_F32, 1, 0, 1, true)); - test_cases.emplace_back(new test_conv_2d_dw({32, 8, 64, 1}, {3, 3, 1, 64}, GGML_TYPE_F32, 2, 1, 1, false)); - test_cases.emplace_back(new test_conv_2d_dw({32, 8, 64, 1}, {3, 3, 1, 64}, GGML_TYPE_F32, 2, 1, 1, true)); - - test_cases.emplace_back(new test_conv_2d_dw({17, 34, 9, 1}, {3, 3, 1, 9}, GGML_TYPE_F16, 1, 0, 1, false)); - test_cases.emplace_back(new test_conv_2d_dw({17, 34, 9, 1}, {3, 3, 1, 9}, GGML_TYPE_F16, 1, 0, 1, true)); - test_cases.emplace_back(new test_conv_2d_dw({32, 8, 64, 1}, {3, 3, 1, 64}, GGML_TYPE_F16, 2, 1, 1, false)); - test_cases.emplace_back(new test_conv_2d_dw({32, 8, 64, 1}, {3, 3, 1, 64}, GGML_TYPE_F16, 2, 1, 1, true)); - - // CONV_3D - auto calc_conv_output_size_3d = [](int64_t ins, int64_t ks, int s, int p, int d) -> int64_t { - return (ins + 2 * p - d * (ks - 1) - 1) / s + 1; - }; - - for (ggml_type kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - for (int N : {1, 2}) { - for (int IC : {1, 3}) { - for (int OC : {1, 4}) { - for (int s0 : {1, 2}) { - for (int p1 : {0, 1}) { - for (int d2 : {1, 2}) { - int64_t IW = 20, IH = 22, ID = 18; - int64_t KW = 3, KH = 3, KD = 3; - int s1 = s0, s2 = s0; - int p0 = p1, p2 = p1; - int d0 = d2, d1 = d2; - - if (calc_conv_output_size_3d(IW, KW, s0, p0, d0) <= 0 || - calc_conv_output_size_3d(IH, KH, s1, p1, d1) <= 0 || - calc_conv_output_size_3d(ID, KD, s2, p2, d2) <= 0) { - continue; - } - test_cases.emplace_back(new test_conv_3d( - N, IC, ID, IH, IW, - OC, KD, KH, KW, - s0, s1, s2, p0, p1, p2, d0, d1, d2, - kernel_type)); - - // Asymmetric kernel and params - int64_t asym_KW = 5, asym_KH = 1, asym_KD = 3; - int asym_s0 = 2, asym_s1 = 1, asym_s2 = 1; - int asym_p0 = 2, asym_p1 = 0, asym_p2 = 1; - int asym_d0 = 1, asym_d1 = 1, asym_d2 = 2; - - if (calc_conv_output_size_3d(IW, asym_KW, asym_s0, asym_p0, asym_d0) <= 0 || - calc_conv_output_size_3d(IH, asym_KH, asym_s1, asym_p1, asym_d1) <= 0 || - calc_conv_output_size_3d(ID, asym_KD, asym_s2, asym_p2, asym_d2) <= 0) { - continue; - } - test_cases.emplace_back(new test_conv_3d( - N, IC, ID, IH, IW, - OC, asym_KD, asym_KH, asym_KW, - asym_s0, asym_s1, asym_s2, asym_p0, asym_p1, asym_p2, asym_d0, asym_d1, asym_d2, - kernel_type)); - } - } - } - } - } - } - // Case with kernel size 1 - test_cases.emplace_back(new test_conv_3d(1, 4, 8, 8, 8, 8, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); - } - - for(uint32_t Cout : {1, 9}){ - for(uint32_t Cin : {1, 7}){ - for(uint32_t K : {1, 3, 1337}){ - for(uint32_t L : {1, 2, 13}){ - for(uint32_t s0: {1, 2, 3}){ - test_cases.emplace_back(new test_conv_transpose_1d({L,Cin,1,1}, {K,Cout,Cin,1}, s0, 0, 1)); - } - } - } - } - } - - test_cases.emplace_back(new test_conv_transpose_1d()); - test_cases.emplace_back(new test_conv_transpose_1d({3,2,1,1}, {2,3,2,1}, 3, 0, 1)); - test_cases.emplace_back(new test_conv_transpose_1d({3,2,1,1}, {2,3,2,1}, 2, 0, 1)); - test_cases.emplace_back(new test_conv_transpose_1d({3,2,1,1}, {2,3,2,1}, 1, 0, 1)); - test_cases.emplace_back(new test_conv_transpose_1d({3,2,1,1}, {3,2,2,1}, 2, 0, 1)); - test_cases.emplace_back(new test_conv_transpose_1d({3,2,1,1}, {3,2,2,1}, 1, 0, 1)); - test_cases.emplace_back(new test_conv_transpose_1d({3,2,1,1}, {3,1,2,1}, 1, 0, 1)); - test_cases.emplace_back(new test_conv_transpose_1d({2,1,1,1}, {3,1,1,1}, 1, 0, 1)); - - for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) { - // ConvTranspose1d expressed as mul_mat + col2im (DAC decoder upsampling) - test_cases.emplace_back(new test_col2im_1d(type, 16, 32, 197, 8, 0)); // kernel = 2*stride - test_cases.emplace_back(new test_col2im_1d(type, 4, 3, 7, 2, 0)); - test_cases.emplace_back(new test_col2im_1d(type, 1, 5, 13, 1, 0)); // stride 1, no overlap - test_cases.emplace_back(new test_col2im_1d(type, 6, 4, 11, 3, 1)); // with cropping - test_cases.emplace_back(new test_col2im_1d(type, 2, 3, 9, 3, 0)); // kernel < stride, gap positions are zeroed - test_cases.emplace_back(new test_col2im_1d(type, 5, 4, 11, 2, 0)); // kernel not a multiple of stride, alternating overlap - test_cases.emplace_back(new test_col2im_1d(type, 8, 4, 13, 4, 2)); // padding = stride/2 (DAC causal cropping) - test_cases.emplace_back(new test_col2im_1d(type, 4, 3, 1, 2, 0)); // single column, pure kernel unfold - test_cases.emplace_back(new test_col2im_1d(type, 16, 1, 197, 8, 0)); // OC = 1, mono output stage - test_cases.emplace_back(new test_col2im_1d(type, 1, 5, 13, 3, 0)); // K = 1 with stride > 1, sparse scatter - test_cases.emplace_back(new test_col2im_1d(type, 8, 2, 3, 2, 5)); // cropping eats most of the signal, T_out = 2 - } - - for (ggml_type kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - test_cases.emplace_back(new test_conv_transpose_2d({3, 2, 3, 1}, {2, 2, 1, 3}, 1, kernel_type)); - test_cases.emplace_back(new test_conv_transpose_2d({10, 10, 9, 1}, {3, 3, 1, 9}, 2, kernel_type)); - test_cases.emplace_back(new test_conv_transpose_2d({129, 63, 35, 1}, {3, 3, 48, 35}, 1, kernel_type)); - } - - test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 500, 1, 1})); - test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 5000, 1, 1})); - - test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32, 1, 1, 1})); - test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32, 513, 1, 1})); - test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {100, 10, 1, 1})); - test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {1024, 10, 1, 1})); - test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {1024, 12, 1, 1})); - test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {2000, 10, 1, 1})); - test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {5438, 3, 1, 1})); - - for (int ne3 : {1, 3}) { // CUDA backward pass only supports ne3 == 1 - test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 1, 1})); - test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {2, 1, 1, 1})); - test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 2, 1, 1})); - test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 2, 1})); - test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 1, 2})); - test_cases.emplace_back(new test_repeat(GGML_TYPE_I32, {10, 5, 4, ne3}, {2, 1, 1, 1})); - test_cases.emplace_back(new test_repeat(GGML_TYPE_I16, {10, 5, 4, ne3}, {1, 1, 1, 2})); - test_cases.emplace_back(new test_repeat(GGML_TYPE_BF16, {10, 5, 4, ne3}, {2, 1, 1, 1})); - } - - for (bool view : {false, true}) { - test_cases.emplace_back(new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {1, 1, 1, 1}, view)); - test_cases.emplace_back(new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {2, 1, 1, 1}, view)); - test_cases.emplace_back(new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {1, 2, 1, 1}, view)); - test_cases.emplace_back(new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {1, 1, 2, 1}, view)); - test_cases.emplace_back(new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {1, 1, 1, 2}, view)); - } - - test_cases.emplace_back(new test_dup(GGML_TYPE_F32)); - test_cases.emplace_back(new test_dup(GGML_TYPE_F16)); - test_cases.emplace_back(new test_dup(GGML_TYPE_I32)); - test_cases.emplace_back(new test_dup(GGML_TYPE_I16)); - test_cases.emplace_back(new test_dup(GGML_TYPE_F32, {10, 10, 5, 1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_dup(GGML_TYPE_F16, {10, 10, 5, 1}, {0, 2, 1, 3})); // dup by rows - test_cases.emplace_back(new test_dup(GGML_TYPE_F32, {10, 10, 5, 1}, {1, 0, 2, 3})); - test_cases.emplace_back(new test_dup(GGML_TYPE_F16, {10, 10, 5, 1}, {1, 0, 2, 3})); // dup dst not-contiguous - test_cases.emplace_back(new test_dup(GGML_TYPE_I16, {10, 8, 3, 1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_dup(GGML_TYPE_I16, {10, 8, 3, 1}, {1, 2, 0, 3})); - - for (int dim = 1; dim < GGML_MAX_DIMS; ++dim) { - test_cases.emplace_back(new test_set(GGML_TYPE_F32, GGML_TYPE_F32, {6, 5, 4, 3}, dim, false)); - test_cases.emplace_back(new test_set(GGML_TYPE_F32, GGML_TYPE_F32, {6, 5, 4, 3}, dim, true)); - } - - for (int dim = 1; dim < GGML_MAX_DIMS; ++dim) { - test_cases.emplace_back(new test_set(GGML_TYPE_I32, GGML_TYPE_I32, {6, 5, 4, 3}, dim, false)); - test_cases.emplace_back(new test_set(GGML_TYPE_I32, GGML_TYPE_I32, {6, 5, 4, 3}, dim, true)); - } - - // same-type copy - for (ggml_type type : all_types) { - const auto nk = ggml_blck_size(type); - - for (int k = 1; k < 4; ++k) { - test_cases.emplace_back(new test_cpy(type, type, {k*nk, 2, 3, 4})); - test_cases.emplace_back(new test_cpy(type, type, {k*nk, 2, 3, 4}, {-1,-1,-1,-1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_cpy(type, type, {k*nk, 2, 3, 4}, {-1,-1,-1,-1}, {0, 3, 1, 2}, {0, 2, 1, 3})); - } - } - - for (ggml_type type_src : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_F32}) { - for (ggml_type type_dst : all_types) { - test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 4, 4, 4})); - test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // cpy by rows - } - } - for (ggml_type type_src : all_types) { - for (ggml_type type_dst : {GGML_TYPE_F32}) { - test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 4, 4, 4})); - test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // cpy by rows - } - } - for (ggml_type type_src : {GGML_TYPE_F16, GGML_TYPE_F32}) { - for (ggml_type type_dst : {GGML_TYPE_F16, GGML_TYPE_F32}) { - test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3})); // cpy not-contiguous - } - } - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_I32, {256, 2, 3, 4})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_I32, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_F32, {256, 2, 3, 4})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_F32, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {256, 4, 3, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 3, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 3, 3}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); - test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {256, 4, 3, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {256, 4, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); - test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {256, 4, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); - test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_I32, {256, 4, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); - test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_I32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {2, 2097121, 1, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {2, 2, 524281, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {128, 2, 3, 1}, {128, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {128, 4, 3, 1})); // strided dst - test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {128, 2, 3, 1}, {128, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {128, 4, 3, 1})); // strided dst - - // CPY - different src/dst shapes (reshaping via CPY) - // Use permutations of {3, 5, 7, 32}. Total elements: 3*5*7*32 = 3360. - // Each src permutation is tested against canonical sorted and reverse dst (skip self). - { - std::array dims = {3, 5, 7, 32}; - std::sort(dims.begin(), dims.end()); - std::array canonical = dims; - std::array reversed = {32, 7, 5, 3}; - for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - std::array cur = dims; - do { - if (cur != canonical) { - test_cases.emplace_back(new test_cpy(type, type, cur, canonical)); - } - if (cur != reversed) { - test_cases.emplace_back(new test_cpy(type, type, cur, reversed)); - } - if (cur[0] == 32 && type == GGML_TYPE_F32) { - if (canonical[0] == 32) { - test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, cur, canonical)); - } - if (reversed[0] == 32) { - test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, cur, reversed)); - } - } - std::next_permutation(cur.begin(), cur.end()); - } while (cur != canonical); - } - } - - for (ggml_type type_dst : { GGML_TYPE_F32, GGML_TYPE_I32, GGML_TYPE_F16, GGML_TYPE_BF16 }) { - for (bool use_view_slice : { true, false }) { - for (std::array ne : std::initializer_list>{ {2, 1, 1, 1}, {2, 1, 3, 5}, - {2, 3, 5, 7}, {1, 4, 4, 1}, {1, 8, 17, 1}, {10, 10, 10, 1} }) { - if (use_view_slice && (type_dst == GGML_TYPE_F16 || type_dst == GGML_TYPE_BF16)) { - continue; // TODO: add after WebGPU is fixed - } - test_cases.emplace_back(new test_cont(type_dst, ne, use_view_slice)); - } - } - } - - auto add_test_bin_bcast = [&](ggml_type type, std::array ne, std::array nr, bool perm1 = false, bool src_overlap = false) { - for (auto op : {ggml_add, ggml_sub, ggml_mul, ggml_div}) { - test_cases.emplace_back(new test_bin_bcast(op, type, ne, nr, 1, perm1, src_overlap)); - } - }; - for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { - for (bool perm1 : {false, true}) { - add_test_bin_bcast(type, {1, 1, 8, 1}, {1, 1, 1, 1}, perm1); - add_test_bin_bcast(type, {1, 1, 1, 1}, {32, 1, 1, 1}, perm1); - add_test_bin_bcast(type, {1, 1, 320, 320}, {1, 1, 1, 1}, perm1); - add_test_bin_bcast(type, {10, 5, 1, 1}, {1, 1, 1, 1}, perm1); - add_test_bin_bcast(type, {10, 5, 4, 1}, {1, 1, 1, 1}, perm1); - add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 1, 1, 1}, perm1); - add_test_bin_bcast(type, {10, 5, 4, 3}, {2, 1, 1, 1}, perm1); - add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 2, 1, 1}, perm1); - add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 1, 2, 1}, perm1); - add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 1, 1, 2}, perm1); - add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 1, 2, 2}, perm1); - add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 2, 2, 2}, perm1); - add_test_bin_bcast(type, {10, 5, 4, 3}, {2, 2, 2, 2}, perm1); - } - - // src_overlap - add_test_bin_bcast(type, {10, 5, 4, 6}, {1, 1, 1, 1}, false, true); - add_test_bin_bcast(type, {10, 5, 4, 5}, {1, 1, 1, 1}, false, true); - add_test_bin_bcast(type, {1, 1, 120, 120}, {1, 1, 1, 1}, false, true); - add_test_bin_bcast(type, {1, 1, 4, 320}, {1, 1, 1, 1}, false, true); - - // test case for k_bin_bcast_unravel in CUDA backend - add_test_bin_bcast(type, {1, 1, 65536, 1}, {256, 1, 1, 1}); - - // stable diffusion - add_test_bin_bcast(type, {1280, 1, 1, 1}, {1, 1, 1, 1}); - add_test_bin_bcast(type, {1280, 1, 1, 1}, {1, 16, 16, 1}); - add_test_bin_bcast(type, {1280, 16, 16, 1}, {1, 1, 1, 1}); - add_test_bin_bcast(type, {1280, 1, 1, 1}, {1, 256, 1, 1}); - add_test_bin_bcast(type, {1, 1, 1280, 1}, {16, 16, 1, 1}); - add_test_bin_bcast(type, {16, 16, 1280, 1}, {1, 1, 1, 1}); - add_test_bin_bcast(type, {1, 1, 1920, 1}, {16, 16, 1, 1}); - add_test_bin_bcast(type, {1, 1, 2560, 1}, {16, 16, 1, 1}); - add_test_bin_bcast(type, {1, 1, 1280, 1}, {32, 32, 1, 1}); - add_test_bin_bcast(type, {1, 1, 1920, 1}, {32, 32, 1, 1}); - add_test_bin_bcast(type, {1, 1, 640, 1}, {32, 32, 1, 1}); - add_test_bin_bcast(type, {5120, 1, 1, 1}, {1, 256, 1, 1}); - add_test_bin_bcast(type, {640, 1, 1, 1}, {1, 1, 1, 1}); - add_test_bin_bcast(type, {64, 262144, 1, 1}, {1, 1, 1, 1}); - //add_test_bin_bcast(type, {3, 3, 2560, 1280}, {1, 1, 1, 1}); - //add_test_bin_bcast(type, {3, 3, 2560, 1280}, {2, 1, 1, 1}); - } - - // single inplace tests, especially important for WebGPU backend since kernels for inplace vs. not are different - test_cases.emplace_back(new test_bin_bcast(ggml_add_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16)); - test_cases.emplace_back(new test_bin_bcast(ggml_mul_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16)); - test_cases.emplace_back(new test_bin_bcast(ggml_sub_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16)); - test_cases.emplace_back(new test_bin_bcast(ggml_div_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16)); - - // fusion - test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {2, 1, 1, 1}, 2)); - test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 2, 1, 1}, 3)); - test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {1, 1, 2, 1}, 4)); - test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 2}, 5)); - test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {1, 1, 2, 2}, 6)); - test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {1, 2, 2, 2}, 7)); - test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {2, 2, 2, 2}, 8)); - test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16)); - - test_cases.emplace_back(new test_scale()); - test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f)); - test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, true)); // inplace test - test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {100, 10, 10, 10}, 2.0f, 1.0f)); - test_cases.emplace_back(new test_softcap(GGML_TYPE_F32, {10, 10, 10, 10}, 50.0f)); - test_cases.emplace_back(new test_silu_back()); - - for (float eps : { 0.0f, 1e-6f, 1e-4f, 1e-1f, 10.f }) { - for (uint32_t n : { 64, 1025 }) { - for (bool v : { false, true }) { - test_cases.emplace_back(new test_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, v, eps)); - test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, v, eps)); - } - test_cases.emplace_back(new test_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, false, eps, true)); - test_cases.emplace_back(new test_rms_norm_back(GGML_TYPE_F32, { n, 5, 4, 3 }, eps)); - test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false)); - test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true)); - test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false, true)); - } - } - - // in-place tests - test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {64, 5, 4, 3}, false, 1e-6f, true)); - - for (float eps : { 0.0f, 1e-6f, 1e-4f, 1e-1f, 1.0f }) { - for (uint32_t n : { 64, 1025 }) { - test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false)); - test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true)); - test_cases.emplace_back(new test_norm_mul_add(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false)); - test_cases.emplace_back(new test_norm_mul_add(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true)); - test_cases.emplace_back(new test_add_rms_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false)); - test_cases.emplace_back(new test_add_rms_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true)); - } - } - for (uint32_t n : {1, 511, 1025, 8192, 33*512}) { - for (bool multi_add : {false, true}) { - test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {n, 1, 1, 1}, 1e-6f, false, multi_add)); - } - test_cases.emplace_back(new test_add_rms_norm(GGML_TYPE_F32, {n, 1, 1, 1}, 1e-6f, false)); - } - - for (auto multi_add : {false, true}) { - for (auto set_rows : {false, true}) { - for (auto rope : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) { - test_cases.emplace_back(new test_rms_norm_mul_rope({768, 1, 1, 1}, 1e-6f, multi_add, set_rows, rope)); - test_cases.emplace_back(new test_rms_norm_mul_rope({768, 3, 1, 1}, 1e-6f, multi_add, set_rows, rope)); - test_cases.emplace_back(new test_rms_norm_mul_rope({768, 3, 5, 1}, 1e-6f, multi_add, set_rows, rope)); - test_cases.emplace_back(new test_rms_norm_mul_rope({128, 32, 2, 1}, 1e-6f, multi_add, set_rows, rope)); - test_cases.emplace_back(new test_rms_norm_mul_rope({128, 4, 2, 1}, 1e-6f, multi_add, set_rows, rope)); - test_cases.emplace_back(new test_rms_norm_mul_rope({128, 32, 50, 1}, 1e-6f, multi_add, set_rows, rope)); - test_cases.emplace_back(new test_rms_norm_mul_rope({128, 4, 50, 1}, 1e-6f, multi_add, set_rows, rope)); - test_cases.emplace_back(new test_rms_norm_mul_rope({8192, 2, 2, 1}, 1e-6f, multi_add, set_rows, rope)); - test_cases.emplace_back(new test_rms_norm_mul_rope({8192, 2, 2, 1}, 1e-6f, multi_add, set_rows, rope)); - } - } - } - for (int64_t d_conv : {3, 4, 9}) { - for (int64_t d_inner: {1024, 1536, 2048}) { - test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1})); - test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {2 * d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1})); - test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {d_conv, d_inner, 4, 1}, {d_conv, d_inner, 1, 1})); - // long token (n_t > 32, exercises the long_token kernel path) - test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {d_conv - 1 + 64, d_inner, 1, 1}, {d_conv, d_inner, 1, 1})); - test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {d_conv - 1 + 64, d_inner, 4, 1}, {d_conv, d_inner, 1, 1})); - } - } - - // fused ssm_conv + (optional) bias_add + silu. The bias-only graph (no silu) is intentionally - // not tested since there's no fusion for that pattern in ggml_cuda_can_fuse. - for (int64_t d_conv : {3, 4, 9}) { - for (int64_t d_inner : {1024, 1536, 2048}) { - for (bool fuse_bias : {false, true}) { - // short token path (n_t <= 32) - test_cases.emplace_back(new test_ssm_conv_bias_silu( - GGML_TYPE_F32, {d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1}, fuse_bias)); - test_cases.emplace_back(new test_ssm_conv_bias_silu( - GGML_TYPE_F32, {2 * d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1}, fuse_bias)); - test_cases.emplace_back(new test_ssm_conv_bias_silu( - GGML_TYPE_F32, {d_conv, d_inner, 4, 1}, {d_conv, d_inner, 1, 1}, fuse_bias)); - // long token path (n_t > 32) - test_cases.emplace_back(new test_ssm_conv_bias_silu( - GGML_TYPE_F32, {d_conv - 1 + 64, d_inner, 1, 1}, {d_conv, d_inner, 1, 1}, fuse_bias)); - test_cases.emplace_back(new test_ssm_conv_bias_silu( - GGML_TYPE_F32, {d_conv - 1 + 64, d_inner, 4, 1}, {d_conv, d_inner, 1, 1}, fuse_bias)); - } - } - } - - test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 16, 1, 1024, 1, 32, 4)); // Mamba-1 - test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 16, 2, 32, 4)); // Mamba-2 - test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 256, 64, 8, 2, 32, 4)); // Falcon-H1 - test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 128, 4, 4, 16, 2, true)); // x/B/C overlap - - test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 1, 1)); - test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 32, 1)); - test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 32, 4)); - test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 128, 4)); - - test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 1, 1)); - test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 32, 1)); - test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 32, 4)); - test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 128, 4)); - - test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 1, 1)); - test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 32, 1)); - test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 32, 4)); - test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 128, 4)); - - // FWHT tests - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 1, 128)); - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 64, 1, 64)); - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 256, 1, 256)); - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 512, 1, 512)); - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 32, 128)); - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 4, 128, {2, 3})); - -#if 0 - // > 4GB A matrix. Too slow to be enabled by default. - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 900000, 3, 2592, {1, 1}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1700000, 96, 2592, {1, 1}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1700000, 3, 2592, {1, 1}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1700000, 1, 2592, {1, 1}, {1, 1})); - - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 128, 128, false, 8192, 2, 5120)); // Llama-4-Maverick-17B-128E-PAB-Q8_0 - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 128, 128, false, 8192, 1, 5120)); // Llama-4-Maverick-17B-128E-PAB-Q8_0 - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8192, 1, 5120, {128, 1}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8192, 512, 5120, {128, 1}, {1, 1})); -#endif - - for (ggml_type type_a : all_types) { - for (int i = 1; i < 10; ++i) { - test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, i, 256, { 1, 1}, {1, 1})); - } - } - - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); - - -#if 0 - { - // Test paths in OpenCL - std::vector ns = {32, 64, 128, 256, 512, 1024, 4096}; - std::vector ks = {896, 1536, 4096}; - for (auto n : ns) { - for (auto k : ks) { - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 1024, n, k, {1, 1}, {1, 1})); - } - } - } -#endif - -#if 1 - for (ggml_type type_a : base_types) { - for (ggml_type type_b : {GGML_TYPE_F32, GGML_TYPE_F16}) { - std::vector ks = { 256 }; - if (ggml_blck_size(type_a) == 1) { - ks.push_back(4); - } - for (auto k : ks) { - // test cases without permutation - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {1, 1}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {1, 1}, {2, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {1, 1}, {1, 2})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {3, 1}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {3, 1}, {2, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {3, 2}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {3, 2}, {2, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {3, 2}, {1, 2})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {3, 2}, {2, 2})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 4, k, {3, 2}, {2, 2})); - - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {1, 1}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {1, 1}, {2, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {1, 1}, {1, 2})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {3, 1}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {3, 1}, {2, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {3, 2}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {3, 2}, {2, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {3, 2}, {1, 2})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {3, 2}, {2, 2})); - - // test cases with permutation - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {2, 3}, {1, 1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {2, 3}, {1, 1}, {0, 1, 3, 2})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {2, 3}, {1, 1}, {0, 3, 2, 1})); - - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 4, k, {2, 3}, {1, 1}, {0, 3, 2, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 8, k, {2, 3}, {1, 1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 8, k, {2, 3}, {1, 1}, {0, 1, 3, 2})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 8, k, {2, 3}, {1, 1}, {0, 3, 2, 1})); - - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {2, 3}, {1, 1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {2, 3}, {1, 1}, {0, 1, 3, 2})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {2, 3}, {1, 1}, {0, 3, 2, 1})); - } - - // test cases with large ne00/ne10 to cover stream-k fixup - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, 1024, {3, 2}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 8, 1024, {3, 2}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, 1024, {3, 2}, {1, 1})); - - // test cases with large batch size - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 8, 256, {1536, 1}, {1, 1})); - } - } - - // BF16 is absent from base_types: add the 3 standard non-contig permutations explicitly - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 1, 256, {2, 3}, {1, 1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 1, 256, {2, 3}, {1, 1}, {0, 1, 3, 2})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 1, 256, {2, 3}, {1, 1}, {0, 3, 2, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 8, 256, {2, 3}, {1, 1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 8, 256, {2, 3}, {1, 1}, {0, 1, 3, 2})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 8, 256, {2, 3}, {1, 1}, {0, 3, 2, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 1, 3, 2})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 3, 2, 1})); - - for (ggml_type type_a : other_types) { - for (ggml_type type_b : {GGML_TYPE_F32}) { - if (ggml_blck_size(type_a) != 256) { - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, ggml_blck_size(type_a), {1, 1}, {1, 1})); - } - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, 256, {1, 1}, {1, 1})); - } - } -#else - // m = a rows - // n = b rows - // k = cols - std::uniform_int_distribution<> dist_m(1, 128); - std::uniform_int_distribution<> dist_n(16, 128); - std::uniform_int_distribution<> dist_k(1, 16); - for (int i = 0; i < 1000; i++) { - for (ggml_type type_a : all_types) { - for (ggml_type type_b : {GGML_TYPE_F32}) { - int m = dist_m(rng); - int n = dist_n(rng); - int k = dist_k(rng) * ggml_blck_size(type_a); - test_cases.emplace_back(new test_mul_mat(type_a, type_b, m, n, k, { 1, 1}, {1, 1})); - } - } - } -#endif - - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 2, 128, { 8, 1}, {1, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 83, 2, 128, { 8, 1}, {4, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 2, 64, { 8, 1}, {4, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 83, 2, 64, { 8, 1}, {4, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 45, 128, { 8, 1}, {4, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 128, 45, 64, { 8, 1}, {4, 1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1056, 1, 193, {1, 1}, {4, 1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1056, 1, 67, {1, 1}, {4, 1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 16, 32, 32, { 1, 1}, {1, 1}, {0, 1, 2, 3}, 64, 3)); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 77, 77, {12,1}, {1,1})); - - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 576, 512, 576, {1,1}, {1,1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 1, 2048, 8192, {1, 1}, {1, 1})); - for (ggml_type type_a : all_types) { - test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1, 64, 256, {1, 1}, {1, 1})); - } - - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 6, 4096, 5120, {1, 1}, {1, 1})); - -#if 0 - // test the mat-mat path for Metal - for (int k = 1; k < 512; ++k) { - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 127, k, {12,1}, {1,1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 127, k, {12,1}, {1,1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 77, k, {12,1}, {1,1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 77, k, {12,1}, {1,1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 128, k, {12,1}, {1,1})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 128, k, {12,1}, {1,1})); - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, false, 50, 200, k)); - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, true, 50, 200, k)); - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 16, 16, false, 50, 200, k)); - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 16, 16, true, 50, 200, k)); - } -#endif - - for (auto bs2 : {1,3}) { - for (auto bs : {1,2,4,8}) { - for (auto nr : {1,4}) { - for (uint32_t m = 0; m < 2; ++m) { - for (uint32_t k = 0; k < 2; ++k) { - for (ggml_type type: {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_F32}) { - test_cases.emplace_back(new test_mul_mat(type, GGML_TYPE_F32, 1056 + m, 1, 128 + k, {bs, bs2}, {nr, 1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_mul_mat(type, GGML_TYPE_F32, 128 + m, 1, 1056 + k, {bs, bs2}, {nr, 1}, {0, 1, 2, 3}, 2*1056 + k)); - } - } - } - } - } - } - - // sycl backend will limit task global_range < MAX_INT - // test case for f16-type-convert-to-fp32 kernel with large k under fp32 compute dtype (occurs in stable-diffusion) - // however this case needs to alloc more memory which may fail in some devices (Intel Arc770, etc.) - // this case is verified (pass) in Intel(R) Data Center GPU Max 1100 (sycl backend) and NV A30 (cuda backend) - // test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 512, 262144, 9216, {1, 1}, {1, 1})); - - // test large experts*tokens - for (bool b : {false, true}) { - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, b, 32, 1024, 16)); - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 2, 2, b, 32, 8192, 64)); - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, b, 50, 200, 64)); - } - - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 1, 1, false, 8, 16, 1)); - test_cases.emplace_back(new test_mul_mat_id_fusion(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, false, 32, 32, 32, 3)); - - // gpt-oss issue with Vulkan mmq_id - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_MXFP4, GGML_TYPE_F32, 32, 2, false, 2880, 32, 2880)); - test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_0, GGML_TYPE_F32, 32, 2, false, 2880, 32, 2880)); - - for (ggml_type type_a : all_types) { - test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 2, false, 64, 16, 3*ggml_blck_size(type_a))); - } - - for (ggml_type type_a : base_types) { - for (ggml_type type_b : {GGML_TYPE_F32 /*, GGML_TYPE_F16 */}) { - for (int n_mats : {4, 8}) { - for (int n_used : {1, 2, 4}) { - for (bool b : {false, true}) { - for (int n : {1, 4, 5, 17, 32, 129}) { - int m = 512; - int k = 256; - test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, n_mats, n_used, b, m, n, k)); - } - } - } - } - } - } - - for (ggml_type type_a : other_types) { - for (ggml_type type_b : {GGML_TYPE_F32 /*, GGML_TYPE_F16 */}) { - for (int n_mats : {4}) { - for (int n_used : {2}) { - for (bool b : {false}) { - for (int n : {1, 32}) { - int m = 512; - int k = 256; - test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, n_mats, n_used, b, m, n, k)); - } - } - } - } - } - } - - for (int bs : {1, 4, 512}) { - for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_K}) { - for (ggml_type type_b : {GGML_TYPE_F32}) { - // test with mul after (ffn_moe_weighted) - test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, type_b, 128, 8, false, 768, bs, 2048, 1, true)); - } - } - } - - for (ggml_type type_a : base_types) { - for (ggml_type type_b : {GGML_TYPE_F32, GGML_TYPE_F16}) { - for (int n : {1, 16}) { - for (int k : {1, 16}) { - for (int bs2 : {1, 3}) { - for (int bs3 : {1, 3}) { - for (int nr2 : {1, 2}) { - for (int nr3 : {1, 2}) { - test_cases.emplace_back(new test_out_prod(type_a, type_b, 256, n, k, {bs2, bs3}, {nr2, nr3})); - } - } - } - } - } - } - } - } - - // ne2 sweep to cover the cublasSgemmStridedBatched path (dps2 == 1, ne2 > 1) - for (int64_t ne2 : {1, 8, 16, 32}) { - test_cases.emplace_back(new test_out_prod(GGML_TYPE_F32, GGML_TYPE_F32, - 256, 16, 16, {ne2, 1}, {1, 1})); - } - - // nr2 sweep to cover the cublasSgemmBatched pointer-array path (dps2 > 1) - for (int64_t nr2 : {8, 16, 32}) { - test_cases.emplace_back(new test_out_prod(GGML_TYPE_F32, GGML_TYPE_F32, - 256, 16, 16, {1, 1}, {nr2, 1})); - } - - // add_id - for (ggml_type type_a : {GGML_TYPE_F32}) { - for (ggml_type type_b : {GGML_TYPE_F32}) { - for (int n_mats : {4, 8}) { - for (int n_used : {1, 2, 4}) { - for (int n_embd : {32, 129}) { - for (int n_token : {1, 32, 129}) { - test_cases.emplace_back(new test_add_id(type_a, type_b, n_embd, n_mats, n_used, n_token)); - } - } - } - } - } - } - - for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { - test_cases.emplace_back(new test_sqr (type)); - test_cases.emplace_back(new test_sqrt (type)); - test_cases.emplace_back(new test_log (type)); - test_cases.emplace_back(new test_sin (type)); - test_cases.emplace_back(new test_cos (type)); - test_cases.emplace_back(new test_clamp (type)); - test_cases.emplace_back(new test_leaky_relu(type)); - test_cases.emplace_back(new test_floor (type)); - test_cases.emplace_back(new test_ceil (type)); - test_cases.emplace_back(new test_round (type)); - test_cases.emplace_back(new test_trunc (type)); - test_cases.emplace_back(new test_sqr (type, {7, 1, 5, 3})); - test_cases.emplace_back(new test_sqr (type, {1024, 1024, 1, 1})); - test_cases.emplace_back(new test_sqrt (type, {7, 1, 5, 3})); - test_cases.emplace_back(new test_sqrt (type, {1024, 1024, 1, 1})); - test_cases.emplace_back(new test_log (type, {7, 1, 5, 3})); - test_cases.emplace_back(new test_log (type, {1024, 1024, 1, 1})); - test_cases.emplace_back(new test_sin (type, {7, 1, 5, 3})); - test_cases.emplace_back(new test_sin (type, {1024, 1024, 1, 1})); - test_cases.emplace_back(new test_cos (type, {7, 1, 5, 3})); - test_cases.emplace_back(new test_cos (type, {1024, 1024, 1, 1})); - test_cases.emplace_back(new test_clamp (type, {7, 1, 5, 3})); - test_cases.emplace_back(new test_clamp (type, {1024, 1024, 1, 1})); - test_cases.emplace_back(new test_leaky_relu(type, {7, 1, 5, 3})); - test_cases.emplace_back(new test_leaky_relu(type, {1024, 1024, 1, 1})); - test_cases.emplace_back(new test_floor (type, {7, 1, 5, 3})); - test_cases.emplace_back(new test_floor (type, {1024, 1024, 1, 1})); - test_cases.emplace_back(new test_ceil (type, {7, 1, 5, 3})); - test_cases.emplace_back(new test_ceil (type, {1024, 1024, 1, 1})); - test_cases.emplace_back(new test_round (type, {7, 1, 5, 3})); - test_cases.emplace_back(new test_round (type, {1024, 1024, 1, 1})); - test_cases.emplace_back(new test_trunc (type, {7, 1, 5, 3})); - test_cases.emplace_back(new test_trunc (type, {1024, 1024, 1, 1})); - } - - test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 1, 1}, 5)); - test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 3, 1}, 5)); - test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 3, 2}, 5)); - -#if 0 - std::uniform_int_distribution<> dist_ne1(1, 50); - int exponent = 1; - while (exponent < (1 << 17)) { - std::uniform_int_distribution<> dist_ne0(exponent, 2*exponent); - - for (int n = 0; n < 10; ++n) { - int64_t ne0 = dist_ne0(rng); - int64_t ne1 = dist_ne1(rng); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, GGML_TYPE_F32, {ne0, ne1, 1, 1}, n/2 == 0, 0.1f, ne0 < 1000 ? 4.0f : 0.0f)); - } - - exponent <<= 1; - } -#endif - for (bool mask : {false, true}) { - for (bool sinks : {false, true}) { - for (float max_bias : {0.0f, 8.0f}) { - if (!mask && max_bias > 0.0f) continue; - for (float scale : {1.0f, 0.1f}) { - for (int64_t ne0 : {16, 1024}) { - for (int64_t ne1 : {16, 1024}) { - if (mask) { - for (ggml_type m_prec : {GGML_TYPE_F32, GGML_TYPE_F16}) { - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0, ne1, 1, 1}, mask, sinks, m_prec, {1, 1}, scale, max_bias)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0-1, ne1-1, 1, 1}, mask, sinks, m_prec, {1, 1}, scale, max_bias)); - - if (ne0 <= 32 && ne1 <= 32) { - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0, ne1, 1, 3}, mask, sinks, m_prec, {3, 1}, scale, max_bias)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0-1, ne1-1, 1, 1}, mask, sinks, m_prec, {2, 3}, scale, max_bias)); - } - } - } else { - /* The precision of mask here doesn't matter as boolean mask is false */ - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0, ne1, 1, 1}, mask, sinks, GGML_TYPE_F32, {1, 1}, scale, max_bias)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0-1, ne1-1, 1, 1}, mask, sinks, GGML_TYPE_F32, {1, 1}, scale, max_bias)); - } - } - } - } - } - // inplace tests - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, mask, sinks, GGML_TYPE_F32, {1, 1}, 0.1f, 0.0f, true)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, mask, sinks, GGML_TYPE_F16, {1, 1}, 0.1f, 0.0f, true)); - } - } - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, true, true, GGML_TYPE_F32, {1, 1}, 0.1f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, true, false, GGML_TYPE_F16, {1, 1}, 0.1f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, false, true, GGML_TYPE_F32, {1, 1}, 0.1f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, true, GGML_TYPE_F32, {1, 1}, 0.1f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, false, GGML_TYPE_F16, {1, 1}, 0.1f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, true, GGML_TYPE_F32, {1, 1}, 0.1f, 8.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, true, GGML_TYPE_F16, {1, 1}, 0.1f, 8.0f)); - - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {200001, 2, 3, 1}, true, true, GGML_TYPE_F32, {1, 1}, 0.1f, 8.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {200001, 2, 3, 1}, true, true, GGML_TYPE_F16, {1, 1}, 0.1f, 8.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {200000, 1, 1, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {200000, 4, 1, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {643251, 3, 1, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); - - for (float max_bias : {0.0f, 8.0f}) { - for (float scale : {1.0f, 0.1f}) { - for (int64_t ne0 : {16, 1024}) { - for (int64_t ne1 : {16, 1024}) { - test_cases.emplace_back(new test_soft_max_back(GGML_TYPE_F32, {ne0, ne1, 1, 1}, scale, max_bias)); - test_cases.emplace_back(new test_soft_max_back(GGML_TYPE_F32, {ne0-1, ne1-1, 1, 1}, scale, max_bias)); - test_cases.emplace_back(new test_soft_max_back(GGML_TYPE_F32, {ne0, ne1, 2, 3}, scale, max_bias)); - } - } - } - } - - for (bool fw : {true, false}) { // fw == forward - bool all = true; - - for (float fs : { 1.0f, 1.4245f }) { - for (float ef : { 0.0f, 0.7465f }) { - for (float af : { 1.0f, 1.4245f }) { - for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - for (bool ff : {false, true}) { // freq_factors - for (float v : { 0, 1 }) { - test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); // llama 7B - - if (all) { - test_cases.emplace_back(new test_rope(type, {128, 40, 2, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); // llama 13B - test_cases.emplace_back(new test_rope(type, {128, 52, 2, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); // llama 30B - test_cases.emplace_back(new test_rope(type, {128, 64, 2, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); // llama 65B - test_cases.emplace_back(new test_rope(type, {16, 16, 8192, 1}, 16, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); - } - - if (all) { - test_cases.emplace_back(new test_rope(type, { 64, 1, 2, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (falcon 7B) - test_cases.emplace_back(new test_rope(type, { 64, 71, 2, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (falcon 7B) - test_cases.emplace_back(new test_rope(type, { 64, 8, 2, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (falcon 40B) - - test_cases.emplace_back(new test_rope(type, { 80, 32, 2, 1}, 20, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); - test_cases.emplace_back(new test_rope(type, { 80, 32, 2, 1}, 32, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); - test_cases.emplace_back(new test_rope(type, { 80, 32, 4, 1}, 32, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); - - test_cases.emplace_back(new test_rope(type, { 80, 32, 2, 1}, 20, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (stablelm) - test_cases.emplace_back(new test_rope(type, { 80, 32, 2, 1}, 32, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (phi-2) - test_cases.emplace_back(new test_rope(type, { 80, 32, 4, 1}, 32, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (phi-2) - test_cases.emplace_back(new test_rope(type, { 16, 16, 8192, 1}, 16, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); - } - - if (all) { - test_cases.emplace_back(new test_rope(type, {128, 12, 2, 1}, 128, GGML_ROPE_TYPE_MROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen2vl 2B) - test_cases.emplace_back(new test_rope(type, {128, 28, 2, 1}, 128, GGML_ROPE_TYPE_MROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen2vl 7B) - test_cases.emplace_back(new test_rope(type, {128, 12, 2, 1}, 20, GGML_ROPE_TYPE_MROPE, 512, fs, ef, af, ff, v, fw)); - test_cases.emplace_back(new test_rope(type, {128, 28, 2, 1}, 32, GGML_ROPE_TYPE_MROPE, 512, fs, ef, af, ff, v, fw)); - test_cases.emplace_back(new test_rope(type, {128, 12, 2, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,imrope (qwen3vl 2B) - test_cases.emplace_back(new test_rope(type, {128, 28, 2, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,imrope (qwen3vl 7B) - test_cases.emplace_back(new test_rope(type, {128, 12, 2, 1}, 20, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); - test_cases.emplace_back(new test_rope(type, {128, 28, 2, 1}, 32, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); - test_cases.emplace_back(new test_rope(type, { 80, 16, 2, 1}, 80, GGML_ROPE_TYPE_VISION, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen2vl ViT) - test_cases.emplace_back(new test_rope(type, {128, 16, 2, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen3vl) - test_cases.emplace_back(new test_rope(type, {16, 16, 8192, 1}, 16, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); - } - - test_cases.emplace_back(new test_rope(type, { 64, 128, 2, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (falcon 40B) - } - - // build_rope_2d-style: ROPE on a non-contiguous view - // that starts at a non-zero offset along dim 0 - // (e.g. gemma4v vision second-half view). - for (int rmode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE, GGML_ROPE_TYPE_VISION }) { - test_cases.emplace_back(new test_rope(type, { 36, 16, 2457, 1}, 36, rmode, 512, fs, ef, af, ff, 2, fw)); - } - } - - all = false; - } - } - } - } - } - - // single inplace test per type/mode/ff - for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE, GGML_ROPE_TYPE_VISION}) { - for (bool ff : {false, true}) { - test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, ff, 0, true, true)); - test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, ff, 1, true, true)); - test_cases.emplace_back(new test_rope(type, {128, 32, 2, 3}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, ff, 1, true, true)); - } - } - } - - for (int v : { 0, 1, 2, 3 }) { - for (int dim : { 0, 1, 2, 3, }) { - test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {11, 12, 13, 14}, 7, dim, v)); - test_cases.emplace_back(new test_concat(GGML_TYPE_F16, {11, 12, 13, 14}, 7, dim, v)); - test_cases.emplace_back(new test_concat(GGML_TYPE_BF16, {11, 12, 13, 14}, 7, dim, v)); - test_cases.emplace_back(new test_concat(GGML_TYPE_I8, {11, 12, 13, 14}, 7, dim, v)); - test_cases.emplace_back(new test_concat(GGML_TYPE_I16, {11, 12, 13, 14}, 7, dim, v)); - test_cases.emplace_back(new test_concat(GGML_TYPE_I32, {11, 12, 13, 14}, 7, dim, v)); - test_cases.emplace_back(new test_concat(GGML_TYPE_I64, {11, 12, 13, 14}, 7, dim, v)); - } - } - - for (ggml_type type_a : { GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 }) { - for (int dim : { 0, 1, 2, 3, }) { - test_cases.emplace_back(new test_concat(type_a, {128, 12, 13, 14}, dim == 0 ? 256 : 7, dim, 0)); - } - } - - for (ggml_sort_order order : {GGML_SORT_ORDER_ASC, GGML_SORT_ORDER_DESC}) { - for (uint32_t i = 4; i <= 1024*1024; i *= 2) { - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {i-1, 1, 1, 1})); - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {i, 1, 1, 1})); - } - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {16, 10, 10, 10}, order)); - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {60, 10, 10, 10}, order)); // qwen - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {1023, 2, 1, 3}, order)); - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {1024, 2, 1, 3}, order)); - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {1025, 2, 1, 3}, order)); - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {1025, 256, 1, 1}, order)); // test ceildiv in CUDA's CUB's DeviceSegmentedSort - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {2047, 2, 1, 3}, order)); - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {2048, 2, 1, 3}, order)); - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {2049, 2, 1, 3}, order)); - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {2, 8, 8192, 1}, order)); // bailingmoe2 (group selection) - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {2048, 512, 1, 1}, order)); // test CUDA dispatching to radix sort for nrows > = 1 in graph mode - } - - for (int n = 1; n < 5; ++n) { - for (int k = 1; k <= n; ++k) { - test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {n, 2, 1, 3}, k, true)); - } - } - for (int i = 0; i < 20; ++i) { - for (int k : {1, 2, 3, 7, 15, 100, 500, 1023, 9999}) { - if (k <= 1< 0.0f) continue; - for (float logit_softcap : {0.0f, 10.0f}) { - if (hsk != 128 && logit_softcap != 0.0f) continue; - for (int nh : { 1, 4 }) { - if (nh == 1 && hsk != 320 && hsk != 576) continue; - for (int nr3 : { 1, 3, }) { - if (hsk > 64 && nr3 > 1) continue; // skip broadcast for large head sizes - for (int nr2 : { 1, 4, 8, 12, 16, 20, 32 }) { - if (nr2 == 8 && hsk != 192) continue; - if (nr2 == 12 && hsk != 128) continue; - if (nr2 == 16 && hsk != 192) continue; - if (nr2 == 20 && (nh != 1 || hsk != 576)) continue; - if (nr2 == 32 && (nh != 1 || hsk != 320)) continue; - //for (int kv : { 1, 17, 31, 33, 61, 113, 65, 127, 129, 130, 255, 260, 371, 380, 407, 512, 1024, }) { - for (int kv : { 113, 512, 1024, }) { - if (nr2 != 1 && kv != 512) continue; - for (int nb : { 1, 3, 32, 75, }) { - for (ggml_prec prec : {GGML_PREC_F32, GGML_PREC_DEFAULT}) { - if (hsk != 128 && prec == GGML_PREC_DEFAULT) continue; - for (ggml_type type_KV : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL}) { - if (type_KV != GGML_TYPE_F16 && hsk != 64 && hsk != 72) continue; - test_cases.emplace_back(new test_flash_attn_ext( - hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV)); - // run fewer test cases permuted - if (mask == true && max_bias == 0.0f && logit_softcap == 0 && kv == 512) { - test_cases.emplace_back(new test_flash_attn_ext( - hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV, {0, 2, 1, 3})); - } - } - } - } - } - } - } - } - } - } - } - } - } - } - - // mixed quant and Q1_0 test cases - test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0)); - test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_F16)); - test_cases.emplace_back(new test_flash_attn_ext(72, 72, 4, {1, 1}, 96, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0)); - test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 96, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F32)); - test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {1, 1}, 256, 1, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0)); - test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {1, 1}, 96, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q1_0, GGML_TYPE_Q1_0)); - test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q1_0, GGML_TYPE_Q4_0)); - test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q1_0)); - test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q1_0, GGML_TYPE_F16)); - - test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, { 10, 5, 4, 3})); - test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, {30000, 1, 1, 1})); - test_cases.emplace_back(new test_cross_entropy_loss_back(GGML_TYPE_F32, { 10, 5, 4, 3})); - test_cases.emplace_back(new test_cross_entropy_loss_back(GGML_TYPE_F32, {30000, 1, 1, 1})); - - test_cases.emplace_back(new test_opt_step_adamw(GGML_TYPE_F32, {10, 5, 4, 3})); - test_cases.emplace_back(new test_opt_step_sgd(GGML_TYPE_F32, {10, 5, 4, 3})); - - for (ggml_type type : base_types) { - for (bool with_gate : {false, true}) { - for (bool use_id : {false, true}) { - for (bool b : {false, true}) { - if (!use_id && b) { - continue; - } - for (bool with_bias : {false, true}) { - if (!with_gate && !with_bias) { - continue; - } - for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) { - if (!with_bias && glu_op == GGML_GLU_OP_SWIGLU_OAI) { - continue; - } - if (!with_gate && glu_op != GGML_GLU_OP_SWIGLU) { - continue; - } - for (bool with_lane_scale : {false, true}) { - if (with_lane_scale && type != GGML_TYPE_NVFP4) { - continue; - } - test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256, - use_id, 16, 8, b, with_bias, with_gate, with_lane_scale)); - test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256, - use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1})); - } - } - } - } - } - } - } - - for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT}) { - for (bool with_norm : {false, true}) { - for (bool bias_probs : {false, true}) { - for (float scale_w : {0.0f, 2.0f}) { - test_cases.emplace_back(new test_topk_moe({8, 22, 1, 1}, 4, with_norm, bias_probs, gate, scale_w)); - test_cases.emplace_back(new test_topk_moe({31, 22, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); - test_cases.emplace_back(new test_topk_moe({32, 22, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); - test_cases.emplace_back(new test_topk_moe({40, 22, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); - test_cases.emplace_back(new test_topk_moe({71, 22, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); - test_cases.emplace_back(new test_topk_moe({128, 1, 1, 1}, 128, with_norm, bias_probs, gate, scale_w)); - test_cases.emplace_back(new test_topk_moe({129, 1, 1, 1}, 128, with_norm, bias_probs, gate, scale_w)); - test_cases.emplace_back(new test_topk_moe({160, 4, 1, 1}, 160, with_norm, bias_probs, gate, scale_w)); - test_cases.emplace_back(new test_topk_moe({288, 22, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); // Used by StepFun 3.7 - } - } - } - } - - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 16, 1, 1)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 16, 1, 1, 1, true, true)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 16, 1, 1, 1, false, true)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 64, 1, 2)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 1)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 2)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 8, 32, 4, 2, 2)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 2, 1, true)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 1, 1, true)); - // KDA (vector gate) - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 1, 1, 1, false, true)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 1, 2, 1, false, true)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 16, 1, 2, 1, false, true)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 32, 4, 1, 1, false, true)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 2, 1, false, true)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 8, 32, 4, 2, 2, false, true)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 2, 1, true, true)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 16, 4, 2, 1, true, true)); - // chunked path: multi-chunk and non-multiple-of-chunk-size (chunk_size=64 GDN, 16 KDA) - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 64, 1)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 127, 1)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 256, 1)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 65, 1)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 100, 1)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 200, 1)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 127, 2)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 64, 1, 1, false, true)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 33, 1, 1, false, true)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 100, 1, 1, false, true)); - - // K > 1: output keeps the last min(n_tokens, K) per-token snapshots, ordered most-recent-first - // (slot 0 = final state, slot s = state s tokens back). - // exact-match cases (K == n_seq_tokens): - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 16, 2, 1, 1, false, false, /*K=*/2)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 32, 4, 1, 1, false, false, /*K=*/4)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 2, 1, false, false, /*K=*/4)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 8, 128, 4, 1, 1, false, false, /*K=*/4)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 2, 1, false, true, /*K=*/4)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 8, 32, 4, 2, 2, false, true, /*K=*/4)); - // overflow: n_tokens > K — only the last K snapshots kept. - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 32, 8, 1, 1, false, false, /*K=*/3)); - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 16, 2, 1, false, false, /*K=*/4)); - -#if 0 - // these tests are disabled to save execution time, sbut they can be handy for debugging - test_cases.emplace_back(new test_llama(2, true)); - test_cases.emplace_back(new test_llama(1)); - test_cases.emplace_back(new test_llama(2)); - test_cases.emplace_back(new test_falcon(1)); - test_cases.emplace_back(new test_falcon(2)); -#endif - - // lightning_indexer - for (int kv : { 256 }) { - for (int bs : { 1, 512 }) { - for (int nh : { 32, 64 }) { - for (auto [ns, nm] : { std::pair{1, 1}, std::pair{4, 4}, std::pair{4, 1} }) { - for (ggml_type type_K : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL}) { - test_cases.emplace_back(new test_lightning_indexer(128, nh, kv, bs, ns, nm, type_K)); - } - } - } - } - } - - return test_cases; -} -#ifdef _MSC_VER -#pragma optimize("", on) -#endif - -// Test cases for performance evaluation: should be representative of real-world use cases -static std::vector> make_test_cases_perf() { - std::vector> test_cases; - - // Conv2d: K=CRS=NPQ=4096 matmul performance - uint32_t iwh_idx = 0; - uint32_t kwh_idx = 1; - uint32_t Cout_idx = 2; - uint32_t Cin_idx = 3; - uint32_t B_idx = 4; - std::vector> cases = { - //{IWH, KWH, Cout, Cin, B} - // K=CRS=NPQ=4096 conv2d matmul performance - {19, 4, 4096, 256, 16}, - // K=128, CRS=128, NPQ=4096 - { 19, 4, 128, 8, 16}, - // K=130, CRS=128, NPQ=4096 - { 19, 4, 130, 8, 16}, - // Edge case: K x CRS is small - { 19, 2, 4, 4, 16}, - // A ConvNet's first layer - { 224, 3, 8, 3, 1 }, - // A ConvNet's first layer with 2x2 convolution, and 1 channel - { 224, 2, 8, 1, 1 }, - // A ConvNet's first layer with 2x2 convolution, and 1 channel, several images in the batch - { 224, 2, 8, 1, 8 }, - // A middle layer of a ConvNet - { 58, 3, 64, 32, 1 }, - // A middle layer of a ConvNet, several images in the batch - { 58, 3, 64, 32, 8 }, - // A deep layer of a ConvNet, several images in the batch - { 16, 3, 512, 128, 8 }, - // High resolution output (large NPQ) - {1536, 3, 64, 32, 1 }, - }; - - for (auto kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - for (auto act_case : cases) { - // Direct CONV_2D - test_cases.emplace_back(new test_conv_2d( - { act_case[iwh_idx], act_case[iwh_idx], act_case[Cin_idx], act_case[B_idx] }, - { act_case[kwh_idx], act_case[kwh_idx], act_case[Cin_idx], act_case[Cout_idx] }, - kernel_type, 1, 1, 0, 0, 1, 1, false)); - } - } - - struct conv3d_perf_case { - int N, IC, ID, IH, IW, OC, KD, KH, KW, s0, s1, s2, p0, p1, p2, d0, d1, d2; - }; - - const std::vector conv3d_cases = { - {1, 320, 8, 38, 26, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, - {1, 1280, 8, 38, 26, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, - {1, 320, 8, 76, 52, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, - {1, 1280, 8, 76, 52, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, - {1, 320, 8, 152, 104, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, -#if 0 - // too slow on some devices - {1, 1280, 8, 152, 104, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, - {1, 320, 4, 304, 208, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, - {1, 640, 4, 304, 208, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, -#endif - }; - - for (ggml_type kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - for (const conv3d_perf_case & c : conv3d_cases) { - test_cases.emplace_back(new test_conv_3d( - c.N, c.IC, c.ID, c.IH, c.IW, - c.OC, c.KD, c.KH, c.KW, - c.s0, c.s1, c.s2, c.p0, c.p1, c.p2, c.d0, c.d1, c.d2, - kernel_type)); - } - } - - test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {4096, 1, 1, 1}, {1, 1, 1, 1})); - test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {4096, 1, 1, 1}, {1, 512, 1, 1})); - - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F16, {512, 3072, 1, 1})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {8192, 512, 2, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {3072, 512, 2, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_Q4_0, {8192, 512, 2, 1})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_F32, {8192, 512, 2, 1})); - - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768*1024, 256, 1, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {0, 0, 0, 0})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768*1024, 256, 1, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {0, 0, 0, 0})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {0, 0, 0, 0})); - test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {0, 0, 0, 0})); - - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768*1024, 256, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768*1024, 256, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); - test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); - test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); - - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {4096, 4096, 5, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {12888, 256, 5, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {77, 4096, 5, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {1024, 1024, 10, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {77, 1024, 10, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {256, 256, 20, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {64, 64, 20, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {77, 64, 20, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); - - test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32, 10, 1, 1})); - test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {1024, 10, 1, 1})); - test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32000, 512, 1, 1})); - - test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {512, 34, 2, 1})); - test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 80, 1, 1})); - test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 80, 4, 1})); - test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 1, 1})); - test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 4, 1})); - - // SNAKE activation fusion at BigVGAN scale (T=7680 = 24 kHz x 320 ms, C=192) - test_cases.emplace_back(new test_snake_fuse(GGML_TYPE_F32, {7680, 192, 1, 1})); - test_cases.emplace_back(new test_snake_fuse(GGML_TYPE_F16, {7680, 192, 1, 1})); - test_cases.emplace_back(new test_snake_fuse(GGML_TYPE_BF16, {7680, 192, 1, 1})); - - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 16416, 1, 128, {8, 1}, {4, 1}, {0, 2, 1, 3})); - test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 128, 1, 16416, {8, 1}, {4, 1}, {0, 1, 2, 3}, 2*16416)); - - // FWHT tests - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 1, 128)); - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 64, 1, 64)); - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 256, 1, 256)); - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 32, 128)); - - test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 64, 64, 4, 4 }, { 32, 64, 4, 4 })); - test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 128, 128, 4, 2 }, { 32, 128, 4, 2 })); - // qwen3next with CHUNK_SIZE 64 - test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 64, 64, 8, 32 }, { 64, 64, 8, 32 })); - // qwen3next with CHUNK_SIZE 128 - test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 128, 128, 4, 32 }, { 128, 128, 4, 32 })); - test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 256, 256, 4, 2 }, { 128, 256, 4, 2 })); - - test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER, GGML_TYPE_F32, { 256, 256, 4, 4 })); - test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER_DIAG, GGML_TYPE_F32, { 1024, 1024, 8, 4 })); - - test_cases.emplace_back(new test_cumsum(GGML_TYPE_F32, { 128, 128, 4, 4 })); - test_cases.emplace_back(new test_cumsum(GGML_TYPE_F32, { 2048, 16, 5, 4 })); - test_cases.emplace_back(new test_cumsum(GGML_TYPE_F32, { 20000, 10, 4, 1 })); - - for (int bs : {1, 2, 3, 4, 5, 8, 512}) { - for (ggml_type type_a : all_types) { - for (ggml_type type_b : {GGML_TYPE_F32}) { - test_cases.emplace_back(new test_mul_mat(type_a, type_b, 4096, bs, 14336, {1, 1}, {1, 1})); - } - } - } - - // qwen3-30b-a3b - for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { - for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) { - for (ggml_type type_b : {GGML_TYPE_F32}) { - test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, 128, 8, false, 768, bs, 2048)); - test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, type_b, 128, 8, false, 768, bs, 2048, 1)); - } - } - } - - for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { - for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) { - for (ggml_type type_b : {GGML_TYPE_F32}) { - test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, 32, 4, false, 1792, bs, 2048)); - test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, type_b, 32, 4, false, 1792, bs, 2048, 1)); - } - } - } - - - // gpt-oss-20b - for (int bs : {1, 4, 8, 512}) { - for (ggml_type type_a : {GGML_TYPE_MXFP4}) { - for (ggml_type type_b : {GGML_TYPE_F32}) { - test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, 32, 4, false, 2880, bs, 2880)); - test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, type_b, 32, 4, false, 2880, bs, 2880, 1)); - } - } - } - - for (int K : {3, 5}) { - for (int IC : {256, 2560}) { - for (int IW_IH : {32, 64, 256}) { - if (IC == 2560 && IW_IH == 256) { - // too big - continue; - } - test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {IW_IH, IW_IH, IC, 1}, {K, K, IC, 1}, 1, 1, 1, 1, 1, 1, true)); - } - } - } - - // Qwen3-VL-8B https://github.com/ggml-org/llama.cpp/issues/17012 - test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 5776, 5776, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); - - test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); - test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); - test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); - test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); - test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); - test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); - - for (int kv : { 4096, 8192, 16384, }) { - for (int hs : { 64, 128, }) { - for (int nr : { 1, 4, }) { - test_cases.emplace_back(new test_flash_attn_ext(hs, hs, 8, {nr, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); - } - } - } - - for (int col : {8192, 16384, 32768, 65536, 131072, 262144, 524288}) { - for (int rows : {1, 4, 16}){ - test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {col, rows, 1, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); - } - } - - test_cases.emplace_back(new test_conv_2d_dw({512, 512, 256, 1}, {3, 3, 1, 256}, GGML_TYPE_F32, 1, 1, 1, false)); - test_cases.emplace_back(new test_conv_2d_dw({512, 512, 256, 1}, {3, 3, 1, 256}, GGML_TYPE_F32, 1, 1, 1, true)); - test_cases.emplace_back(new test_conv_2d_dw({112, 112, 32, 1}, {3, 3, 1, 32}, GGML_TYPE_F32, 1, 1, 1, false)); - test_cases.emplace_back(new test_conv_2d_dw({112, 112, 32, 1}, {3, 3, 1, 32}, GGML_TYPE_F32, 1, 1, 1, true)); - test_cases.emplace_back(new test_conv_2d_dw({56, 56, 128, 1}, {5, 5, 1, 128}, GGML_TYPE_F32, 2, 2, 1, false)); - test_cases.emplace_back(new test_conv_2d_dw({56, 56, 128, 1}, {5, 5, 1, 128}, GGML_TYPE_F32, 2, 2, 1, true)); - - for (ggml_type kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - test_cases.emplace_back(new test_conv_transpose_2d({256, 256, 256, 1}, {3, 3, 16, 256}, 1, kernel_type)); - test_cases.emplace_back(new test_conv_transpose_2d({16, 16, 16, 1}, {3, 3, 8, 16}, 1, kernel_type)); - test_cases.emplace_back(new test_conv_transpose_2d({10, 10, 9, 1}, {3, 3, 1, 9}, 2, kernel_type)); - } - - // Memory bound overlap-add of the GEMM + col2im_1d transposed conv path, real vocoder stage shapes - test_cases.emplace_back(new test_col2im_1d(GGML_TYPE_F32, 16, 512, 2048, 8, 0)); - test_cases.emplace_back(new test_col2im_1d(GGML_TYPE_F32, 4, 128, 65536, 2, 0)); - test_cases.emplace_back(new test_col2im_1d(GGML_TYPE_F16, 16, 512, 2048, 8, 0)); - - test_cases.emplace_back(new test_mean(GGML_TYPE_F32, {256, 256, 3, 1})); - - - for (int n_token : {1, 512}) { - test_cases.emplace_back(new test_add_id(GGML_TYPE_F32, GGML_TYPE_F32, 2880, 128, 4, n_token)); - test_cases.emplace_back(new test_add_id(GGML_TYPE_F32, GGML_TYPE_F32, 2880, 32, 4, n_token)); - } - - for (bool fw : {true, false}) { // fw == forward - for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - for (bool ff : {false, true}) { // freq_factors - for (float v : { 0, 1 }) { - test_cases.emplace_back(new test_rope(type, {128, 32, 512, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // llama 7B - test_cases.emplace_back(new test_rope(type, {128, 64, 512, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // llama 65B - test_cases.emplace_back(new test_rope(type, { 80, 32, 512, 1}, 20, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // neox (stablelm) - test_cases.emplace_back(new test_rope(type, { 64, 8, 512, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // neox (falcon 40B) - test_cases.emplace_back(new test_rope(type, {128, 12, 512, 1}, 128, GGML_ROPE_TYPE_MROPE, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // rope_multi,m-rope (qwen2vl 2B) - test_cases.emplace_back(new test_rope(type, {128, 12, 512, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // rope_multi,imrope (qwen3vl 2B) - test_cases.emplace_back(new test_rope(type, { 80, 16, 2, 1}, 80, GGML_ROPE_TYPE_VISION, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // rope_multi,m-rope (qwen2vl ViT) - } - } - } - } - - std::vector> reduce_rows_cases = { - { 8192, 1, 1, 1 }, - { 8192, 8192, 1, 1 }, - { 128, 8192, 1, 1 }, - }; - - for (auto it: reduce_rows_cases){ - test_cases.emplace_back(new test_mean(GGML_TYPE_F32, it)); - test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, it)); - test_cases.emplace_back(new test_sum(GGML_TYPE_F32, it)); - } - - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {65000, 16, 1, 1})); - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {200000, 1, 1, 1})); - test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {200000, 16, 1, 1})); - - test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {2, 1, 1, 1}, 1)); - for (auto k : {1, 10, 40, 400}) { - for (auto nrows : {1, 16}) { - for (auto cols : {k, 1000, 65000, 200000}) { - test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {cols, nrows, 1, 1}, k)); - } - } - } - - for (auto nrows : {1, 4, 8, 16}) { - for (auto cols : {128, 1024, 4096, 8192, 16384, 32768, 65536, 131072, 200000, 2000000}) { - test_cases.emplace_back(new test_cumsum(GGML_TYPE_F32, {cols, nrows, 1, 1})); - } - } - - // Examples from granite-4.0-h-1b/ggml-model-Q8_0.gguf - test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {515, 3328, 1, 1}, {4, 3328, 1, 1})); // prefill - test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {937, 8192, 1, 1}, {4, 8192, 1, 1})); // prefill - test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {4, 3328, 1, 1}, {4, 3328, 1, 1})); // generate - test_cases.emplace_back(new test_ssm_conv_bias_silu(GGML_TYPE_F32, {515, 3328, 1, 1}, {4, 3328, 1, 1}, true)); // prefill - test_cases.emplace_back(new test_ssm_conv_bias_silu(GGML_TYPE_F32, {4, 3328, 1, 1}, {4, 3328, 1, 1}, true)); // generate - test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 48, 1, 512, 1)); // prefill - test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 48, 1, 1, 1)); // generate - - // acc - test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 1, 1}, {256, 16, 1, 1}, -1)); - test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {256, 16, 2, 3}, -1)); - test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {128, 16, 2, 3}, -1)); - test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {256, 16, 2, 3}, 1)); - test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {128, 16, 2, 3}, 2)); - test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {64, 16, 2, 3}, 3)); - - // GATED_DELTA_NET: realistic model configurations - // TG: n_seq_tokens=1 (autoregressive) - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1)); // Qwen3.5-like: 32 heads, d=128 - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 64, 1, 1)); // smaller model - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1, 1, false, true)); // KDA - // PP: n_seq_tokens=64,256 (prompt processing) - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 64, 1)); // PP-64 - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 256, 1)); // PP-256 - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 512, 1)); // PP-512 - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1024, 1)); // PP-1024 - // Small model configs (fewer heads = less GPU occupancy for autoregressive) - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1)); // 4h PP-64 - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 256, 1)); // 4h PP-256 - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 512, 1)); // 4h PP-512 - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1024, 1)); // 4h PP-1024 - test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 64, 1, 1, false, true)); // KDA PP-64 - - // lightning_indexer - for (int kv : { 256, 4096, 65536 }) { - for (int bs : { 1, 512, 2048 }) { - for (int nh : { 32, 64 }) { - for (int ns : { 1, 4 }) { - for (ggml_type type_K : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL}) { - test_cases.emplace_back(new test_lightning_indexer(128, nh, kv, bs, ns, ns, type_K)); - } - } - } - } - } - - return test_cases; -} - -static std::vector> make_test_cases_from_file(const char * path) { - std::ifstream f(path); - - if (!f.is_open()) { - throw std::runtime_error("Unable to read test file"); - } - - std::vector> test_cases; - - std::string line; - - while (std::getline(f, line)) { - std::istringstream iss(line); - - ggml_op op; - ggml_type type; - std::array ne; - std::array op_params = {}; - std::string name; - uint64_t tmp; - - iss >> tmp; - op = (ggml_op)tmp; - iss >> tmp; - type = (ggml_type)tmp; - - for (size_t i = 0; i < 4; i++) { - iss >> ne[i]; - } - - iss >> tmp; - for (size_t i = 0; i < tmp && i < op_params.size(); i++) { - iss >> op_params[i]; - } - - iss >> tmp; - - size_t num_src = std::min((uint64_t)GGML_MAX_SRC, tmp); - std::vector sources(num_src); - for (size_t i = 0; i < num_src; i++) { - input_tensor& src = sources[i]; - iss >> tmp; - src.type = (ggml_type)tmp; - - for (size_t i = 0; i < 4; i++) { - iss >> src.ne[i]; - } - for (size_t i = 0; i < 4; i++) { - iss >> src.nb[i]; - } - } - - iss >> name; - - if (name.length() == 1 && name[0] == '-') { - name = ""; - } - - test_cases.emplace_back(new test_generic_op(op, type, ne, op_params, sources, std::move(name))); - } - - return test_cases; -} - -static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mode mode, const char * op_names_filter, const char * params_filter, - printer * output_printer, const char * test_file_path, int parallel_workers) { - auto filter_test_cases = [](std::vector> & test_cases, const char * params_filter) { - if (params_filter == nullptr) { - return; - } - - std::regex params_filter_regex(params_filter); - - for (auto it = test_cases.begin(); it != test_cases.end();) { - if (!std::regex_search((*it)->vars(), params_filter_regex)) { - it = test_cases.erase(it); - continue; - } - - it++; - } - }; - - std::vector> test_cases; - - if (test_file_path == nullptr) { - switch (mode) { - case MODE_TEST: - case MODE_GRAD: - case MODE_SUPPORT: - test_cases = make_test_cases_eval(); - break; - case MODE_PERF: - test_cases = make_test_cases_perf(); - break; - } - } else { - test_cases = make_test_cases_from_file(test_file_path); - } - - filter_test_cases(test_cases, params_filter); - - if (mode == MODE_TEST) { - ggml_backend_ptr backend_cpu(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL)); - if (backend_cpu == NULL) { - test_operation_info info("", "", "CPU"); - info.set_error("backend", "Failed to initialize CPU backend"); - output_printer->print_operation(info); - return false; - } - // Use reference implementation on the CPU backend for comparison - using ggml_backend_cpu_set_use_ref_t = void (*)(ggml_backend_t, bool); - auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu.get())); - auto * set_use_ref = (ggml_backend_cpu_set_use_ref_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_use_ref"); - if (set_use_ref) { - set_use_ref(backend_cpu.get(), true); - } - - std::atomic n_ok = 0; - std::atomic tests_run = 0; - std::vector failed_tests; - std::mutex failed_tests_mutex; - - // Each worker grabs a chunk of cases at a time. The chunk shrinks as we - // run out of work so that a few slow tests at the tail get spread across - // workers instead of landing on one unlucky thread. - constexpr size_t MAX_TESTS_PER_ITER = 100; - std::atomic test_idx = 0; - - const auto & next_chunk = [&](size_t & my_begin, size_t & my_end) { - const size_t cur = test_idx.load(std::memory_order_relaxed); - const size_t remaining = cur < test_cases.size() ? test_cases.size() - cur : 0; - const size_t chunk = std::max(1, std::min(MAX_TESTS_PER_ITER, remaining / parallel_workers)); - my_begin = test_idx.fetch_add(chunk); - my_end = std::min(my_begin + chunk, test_cases.size()); - }; - - const auto & run_tests = [&](ggml_backend_t b, ggml_backend_t b_cpu) { - size_t my_begin, my_end; - next_chunk(my_begin, my_end); - while (my_begin < test_cases.size()) { - for (size_t i = my_begin; i < my_end; ++i) { - auto & test = test_cases[i]; - test_status_t status = test->eval(b, b_cpu, op_names_filter, output_printer); - if (status == test_status_t::SKIPPED || status == test_status_t::NOT_SUPPORTED) { - continue; - } - tests_run++; - if (status == test_status_t::OK) { - n_ok++; - } else if (status == test_status_t::FAIL) { - std::lock_guard guard(failed_tests_mutex); - failed_tests.push_back(test->current_op_name + "(" + test->vars() + ")"); - } - } - next_chunk(my_begin, my_end); - } - }; - - if (parallel_workers <= 1) { - // Reuse the outer backend / backend_cpu so we don't pay an - // extra CPU backend init. - run_tests(backend, backend_cpu.get()); - } else { - std::atomic workers_started = 0; - - const auto & eval_worker = [&]() { - ggml_backend_ptr b(ggml_backend_dev_init(dev, NULL)); - if (b == NULL) { - return; - } - - ggml_backend_ptr b_cpu(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL)); - if (b_cpu == NULL) { - return; - } - - if (set_use_ref) { - set_use_ref(b_cpu.get(), true); - } - workers_started++; - run_tests(b.get(), b_cpu.get()); - }; - - std::vector threads; - threads.reserve(parallel_workers); - for (int i = 0; i < parallel_workers; ++i) { - threads.emplace_back(eval_worker); - } - for (auto & t : threads) { - t.join(); - } - - if (workers_started == 0 && !test_cases.empty()) { - return false; - } - } - - output_printer->print_summary(test_summary_info(n_ok, tests_run, false)); - output_printer->print_failed_tests(failed_tests); - - return n_ok == tests_run; - } - - if (mode == MODE_GRAD) { - test_cases.erase( - std::remove_if(test_cases.begin(), test_cases.end(), [](const std::unique_ptr & tc) { - return tc->run_whole_graph(); - }), - test_cases.end() - ); - - size_t n_ok = 0; - for (auto & test : test_cases) { - if (test->eval_grad(backend, op_names_filter, output_printer)) { - n_ok++; - } - } - output_printer->print_summary(test_summary_info(n_ok, test_cases.size(), false)); - - return n_ok == test_cases.size(); - } - - if (mode == MODE_PERF) { - for (auto & test : test_cases) { - test->eval_perf(backend, op_names_filter, output_printer); - } - return true; - } - - if (mode == MODE_SUPPORT) { - // Filter out fusion cases - test_cases.erase( - std::remove_if(test_cases.begin(), test_cases.end(), [](const std::unique_ptr & tc) { - return tc->run_whole_graph(); - }), - test_cases.end() - ); - - for (auto & test : test_cases) { - test->eval_support(backend, op_names_filter, output_printer); - } - return true; - } - - GGML_ABORT("fatal error"); -} - -static void list_all_ops() { - printf("GGML operations:\n"); - std::set all_ops; - - for (int i = 1; i < GGML_OP_COUNT; i++) { - all_ops.insert(ggml_op_name((enum ggml_op)i)); - } - for (int i = 0; i < GGML_UNARY_OP_COUNT; i++) { - all_ops.insert(ggml_unary_op_name((enum ggml_unary_op)i)); - } - for (int i = 0; i < GGML_GLU_OP_COUNT; i++) { - all_ops.insert(ggml_glu_op_name((enum ggml_glu_op)i)); - } - for (const auto & op : all_ops) { - printf(" %s\n", op.c_str()); - } - printf("\nTotal: %zu operations\n", all_ops.size()); -} - -static void show_test_coverage() { - std::set all_ops; - for (int i = 1; i < GGML_OP_COUNT; i++) { - auto op = (enum ggml_op)i; - if (op == GGML_OP_VIEW || - op == GGML_OP_RESHAPE || - op == GGML_OP_PERMUTE || - op == GGML_OP_TRANSPOSE || - op == GGML_OP_CONT || - op == GGML_OP_GLU || - op == GGML_OP_UNARY) { - continue; - } - all_ops.insert(ggml_op_name(op)); - } - for (int i = 0; i < GGML_UNARY_OP_COUNT; i++) { - all_ops.insert(ggml_unary_op_name((enum ggml_unary_op)i)); - } - for (int i = 0; i < GGML_GLU_OP_COUNT; i++) { - all_ops.insert(ggml_glu_op_name((enum ggml_glu_op)i)); - } - auto test_cases = make_test_cases_eval(); - // Filter out fusion cases - test_cases.erase( - std::remove_if(test_cases.begin(), test_cases.end(), [](const std::unique_ptr & tc) { - return tc->run_whole_graph(); - }), - test_cases.end() - ); - - std::set tested_ops; - - ggml_init_params params = { - /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead(), - /* .mem_base = */ NULL, - /* .no_alloc = */ true, - }; - - for (auto & test_case : test_cases) { - ggml_context_ptr ctx(ggml_init(params)); - if (ctx) { - test_case->mode = MODE_TEST; - ggml_tensor * out = test_case->build_graph(ctx.get()); - if (out && out->op != GGML_OP_NONE) { - if (out->op == GGML_OP_UNARY) { - tested_ops.insert(ggml_unary_op_name(ggml_get_unary_op(out))); - } else if (out->op == GGML_OP_GLU) { - tested_ops.insert(ggml_glu_op_name(ggml_get_glu_op(out))); - } else { - tested_ops.insert(ggml_op_name(out->op)); - } - } - } - } - std::set covered_ops; - std::set uncovered_ops; - for (const auto & op : all_ops) { - if (tested_ops.count(op) > 0) { - covered_ops.insert(op); - } else { - uncovered_ops.insert(op); - } - } - - printf("Operations covered by tests (%zu):\n", covered_ops.size()); - for (const auto & op : covered_ops) { - printf(" ✓ %s\n", op.c_str()); - } - printf("\nOperations without tests (%zu):\n", uncovered_ops.size()); - for (const auto & op : uncovered_ops) { - printf(" ✗ %s\n", op.c_str()); - } - - printf("\nCoverage Summary:\n"); - printf(" Total operations: %zu\n", all_ops.size()); - printf(" Tested operations: %zu\n", covered_ops.size()); - printf(" Untested operations: %zu\n", uncovered_ops.size()); - printf(" Coverage: %.1f%%\n", (double)covered_ops.size() / all_ops.size() * 100.0); -} - -static void usage(char ** argv) { - printf("Usage: %s [mode] [-o ] [-b ] [-p ] [--output ] [--list-ops]", argv[0]); - printf(" [--show-coverage] [--test-file ] [-j ]\n"); - printf(" valid modes:\n"); - printf(" - test (default, compare with CPU backend for correctness)\n"); - printf(" - grad (compare gradients from backpropagation with method of finite differences)\n"); - printf(" - perf (performance evaluation)\n"); - printf(" - support (probe backend operation support)\n"); - printf(" op names for -o are as given by ggml_op_desc() (e.g. ADD, MUL_MAT, etc),\n"); - printf(" optionally including the full test case string (e.g. \"ADD(type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1)\")\n"); - printf(" --output specifies output format (default: console, options: console, sql, csv)\n"); - printf(" --list-ops lists all available GGML operations\n"); - printf(" --show-coverage shows test coverage\n"); - printf(" --test-file reads test operators from a test file generated by test-export-graph-ops\n"); - printf(" -j runs tests using parallel worker threads (default: 1, test mode only)\n"); -} - -int main(int argc, char ** argv) { - test_mode mode = MODE_TEST; - output_formats output_format = CONSOLE; - const char * op_names_filter = nullptr; - const char * backend_filter = nullptr; - const char * params_filter = nullptr; - const char * test_file_path = nullptr; - int parallel_workers = 1; - - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "test") == 0) { - mode = MODE_TEST; - } else if (strcmp(argv[i], "perf") == 0) { - mode = MODE_PERF; - } else if (strcmp(argv[i], "grad") == 0) { - mode = MODE_GRAD; - } else if (strcmp(argv[i], "support") == 0) { - mode = MODE_SUPPORT; - } else if (strcmp(argv[i], "-o") == 0) { - if (i + 1 < argc) { - op_names_filter = argv[++i]; - } else { - usage(argv); - return 1; - } - } else if (strcmp(argv[i], "-b") == 0) { - if (i + 1 < argc) { - backend_filter = argv[++i]; - } else { - usage(argv); - return 1; - } - } else if (strcmp(argv[i], "-p") == 0) { - if (i + 1 < argc) { - params_filter = argv[++i]; - } else { - usage(argv); - return 1; - } - } else if (strcmp(argv[i], "--output") == 0) { - if (i + 1 < argc) { - if (!output_format_from_str(argv[++i], output_format)) { - usage(argv); - return 1; - } - } else { - usage(argv); - return 1; - } - } else if (strcmp(argv[i], "--list-ops") == 0) { - list_all_ops(); - return 0; - } else if (strcmp(argv[i], "--show-coverage") == 0) { - show_test_coverage(); - return 0; - } else if (strcmp(argv[i], "--test-file") == 0) { - if (i + 1 < argc) { - test_file_path = argv[++i]; - } else { - usage(argv); - return 1; - } - } else if (strcmp(argv[i], "-j") == 0) { - if (i + 1 < argc) { - parallel_workers = atoi(argv[++i]); - if (parallel_workers < 1) { - usage(argv); - return 1; - } - } else { - usage(argv); - return 1; - } - } else { - usage(argv); - return 1; - } - } - - // load and enumerate backends - ggml_backend_load_all(); - - // Create printer for output format - std::unique_ptr output_printer = create_printer(output_format); - if (output_printer) { - output_printer->print_header(); - } - - output_printer->print_testing_start(testing_start_info(ggml_backend_dev_count())); - - size_t n_ok = 0; - - for (size_t i = 0; i < ggml_backend_dev_count(); i++) { - ggml_backend_dev_t dev = ggml_backend_dev_get(i); - - if (backend_filter != NULL && strcmp(backend_filter, ggml_backend_dev_name(dev)) != 0) { - output_printer->print_backend_init( - backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping")); - n_ok++; - continue; - } - - if (backend_filter == NULL && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU && mode != MODE_GRAD) { - output_printer->print_backend_init(backend_init_info( - i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping CPU backend")); - n_ok++; - continue; - } - - ggml_backend_ptr backend(ggml_backend_dev_init(dev, NULL)); - GGML_ASSERT(backend != NULL); - - ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); - auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); - if (ggml_backend_set_n_threads_fn) { - // TODO: better value for n_threads - ggml_backend_set_n_threads_fn(backend.get(), N_THREADS); - } - - size_t free, total; // NOLINT - ggml_backend_dev_memory(dev, &free, &total); - output_printer->print_backend_init(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), - false, "", ggml_backend_dev_description(dev), - total / 1024 / 1024, free / 1024 / 1024, true)); - - bool ok = test_backend(backend.get(), dev, mode, op_names_filter, params_filter, output_printer.get(), test_file_path, parallel_workers); - - if (ok) { - n_ok++; - } - output_printer->print_backend_status( - backend_status_info(ggml_backend_name(backend.get()), ok ? test_status_t::OK : test_status_t::FAIL)); - } - - ggml_quantize_free(); - - if (output_printer) { - output_printer->print_footer(); - } - - output_printer->print_overall_summary( - overall_summary_info(n_ok, ggml_backend_dev_count(), n_ok == ggml_backend_dev_count())); - - if (n_ok != ggml_backend_dev_count()) { - return 1; - } - - return 0; -} From be1667077161b7782fca67e34a36125b466d7215 Mon Sep 17 00:00:00 2001 From: Vidas Date: Mon, 29 Sep 2025 14:56:57 +0300 Subject: [PATCH 02/12] ggml-et: Add GET_ROWS op --- ggml/src/ggml-et/ggml-et-cpu-compare.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ggml/src/ggml-et/ggml-et-cpu-compare.cpp b/ggml/src/ggml-et/ggml-et-cpu-compare.cpp index 15c710cc2d5f..ad9fd9f127e4 100644 --- a/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +++ b/ggml/src/ggml-et/ggml-et-cpu-compare.cpp @@ -319,6 +319,9 @@ bool ggml_et_cpu_compare_compute_and_check(ggml_et_cpu_compare_ctx * ct ctx->cpu_dst = ggml_rms_norm(ctx->ggml_ctx, ctx->cpu_src0, eps); } break; + case GGML_OP_GET_ROWS: + ctx->cpu_dst = ggml_get_rows(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1); + break; default: GGML_LOG_ERROR("ET: Unsupported operation %s for CPU comparison\n", ggml_op_name(op)); return false; From f20903f7e567ad377dc63cab03168a9396445fb8 Mon Sep 17 00:00:00 2001 From: Vidas Date: Sun, 19 Oct 2025 19:38:43 +0300 Subject: [PATCH 03/12] ggml-et: Fix CONT Reorder from contiguous write to read with atomic stores. --- ggml/src/ggml-et/et-kernels/src/platform.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ggml/src/ggml-et/et-kernels/src/platform.h b/ggml/src/ggml-et/et-kernels/src/platform.h index cbec4c98d741..78c41b8b5a18 100644 --- a/ggml/src/ggml-et/et-kernels/src/platform.h +++ b/ggml/src/ggml-et/et-kernels/src/platform.h @@ -148,6 +148,19 @@ static inline void atomic_store_f16(volatile uint16_t * addr, uint16_t value) { et_global_store_hw(addr, value); } +// Atomic store for F16 values to global memory +// Uses ET hardware's custom shg instruction (store halfword global) +// This ensures cache coherency when multiple threads write to nearby addresses +// Address must be 16-bit aligned +static inline void atomic_store_f16(volatile uint16_t* addr, uint16_t value) { + __asm__ volatile( + "shg %1, (%0)" + : + : "r"(addr), "r"(value) + : "memory" + ); +} + //****************************************************************************** // Barrier Primitives // From 15db09c84167bdc14e02db3b394e44955d396fbf Mon Sep 17 00:00:00 2001 From: marty1885 Date: Wed, 25 Feb 2026 07:04:17 +0000 Subject: [PATCH 04/12] wip: message default message cleanup --- ggml/src/ggml-et/ggml-et-logger.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 ggml/src/ggml-et/ggml-et-logger.h diff --git a/ggml/src/ggml-et/ggml-et-logger.h b/ggml/src/ggml-et/ggml-et-logger.h new file mode 100644 index 000000000000..8a26b74ba621 --- /dev/null +++ b/ggml/src/ggml-et/ggml-et-logger.h @@ -0,0 +1,13 @@ +#pragma once + +#include "ggml-backend.h" + +// internal API +void ggml_et_log_internal(ggml_log_level level, const char* file, int line, const char* fmt, ...); + +#define ET_LOG_INTERNAL(level, ...) ggml_et_log_internal(level, __FILE_NAME__, __LINE__, __VA_ARGS__) +#define ET_LOG(...) ET_LOG_INTERNAL(GGML_LOG_LEVEL_TRACE, __VA_ARGS__) +#define ET_LOG_DEBUG(...) ET_LOG_INTERNAL(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) +#define ET_LOG_INFO(...) ET_LOG_INTERNAL(GGML_LOG_LEVEL_INFO, __VA_ARGS__) +#define ET_LOG_WARN(...) ET_LOG_INTERNAL(GGML_LOG_LEVEL_WARN, __VA_ARGS__) +#define ET_LOG_ERROR(...) ET_LOG_INTERNAL(GGML_LOG_LEVEL_ERROR, __VA_ARGS__) From ac380f5c8c9ad84eb59be70cc4d3345021cd8699 Mon Sep 17 00:00:00 2001 From: marty1885 Date: Thu, 26 Feb 2026 08:33:29 +0000 Subject: [PATCH 05/12] chore: message cleanups --- ggml/src/ggml-et/ggml-et-logger.h | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 ggml/src/ggml-et/ggml-et-logger.h diff --git a/ggml/src/ggml-et/ggml-et-logger.h b/ggml/src/ggml-et/ggml-et-logger.h deleted file mode 100644 index 8a26b74ba621..000000000000 --- a/ggml/src/ggml-et/ggml-et-logger.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include "ggml-backend.h" - -// internal API -void ggml_et_log_internal(ggml_log_level level, const char* file, int line, const char* fmt, ...); - -#define ET_LOG_INTERNAL(level, ...) ggml_et_log_internal(level, __FILE_NAME__, __LINE__, __VA_ARGS__) -#define ET_LOG(...) ET_LOG_INTERNAL(GGML_LOG_LEVEL_TRACE, __VA_ARGS__) -#define ET_LOG_DEBUG(...) ET_LOG_INTERNAL(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) -#define ET_LOG_INFO(...) ET_LOG_INTERNAL(GGML_LOG_LEVEL_INFO, __VA_ARGS__) -#define ET_LOG_WARN(...) ET_LOG_INTERNAL(GGML_LOG_LEVEL_WARN, __VA_ARGS__) -#define ET_LOG_ERROR(...) ET_LOG_INTERNAL(GGML_LOG_LEVEL_ERROR, __VA_ARGS__) From 9cb27fdb1c3ebdff066d52e8c61a85be3b1c4f90 Mon Sep 17 00:00:00 2001 From: marty1885 Date: Thu, 26 Feb 2026 09:15:14 +0000 Subject: [PATCH 06/12] migrate to use platform provided functions --- ggml/src/ggml-et/et-kernels/src/platform.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ggml/src/ggml-et/et-kernels/src/platform.h b/ggml/src/ggml-et/et-kernels/src/platform.h index 78c41b8b5a18..6825cf9865d6 100644 --- a/ggml/src/ggml-et/et-kernels/src/platform.h +++ b/ggml/src/ggml-et/et-kernels/src/platform.h @@ -13,6 +13,8 @@ #include "etsoc/isa/hart.h" #include +#include "etsoc/isa/hart.h" +#include "etsoc/common/utils.h" #define SOC_MINIONS_PER_SHIRE 32 #define NUM_HARTS_PER_MINION 2 From 4a5751b8043d8e2dd882f54849bbcaf8dcafbaec Mon Sep 17 00:00:00 2001 From: marty1885 Date: Thu, 26 Feb 2026 09:27:41 +0000 Subject: [PATCH 07/12] cmake back into subdir --- ggml/src/ggml-et/ggml-et-kernels.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml/src/ggml-et/ggml-et-kernels.cpp b/ggml/src/ggml-et/ggml-et-kernels.cpp index 3e119283e082..fdda362b4dd6 100644 --- a/ggml/src/ggml-et/ggml-et-kernels.cpp +++ b/ggml/src/ggml-et/ggml-et-kernels.cpp @@ -96,6 +96,10 @@ static bool ggml_et_uberkernel_ensure_slot_capacity(ggml_backend_et_uberkernel_s return slot.device_insts != nullptr && slot.device_params != nullptr; } +#define ET_TRACE_DECODER_IMPL +#include +#include + // Get embedded kernel data by name static std::vector ggml_et_get_embedded_kernel(const std::string & kernel_name) { auto it = ggml_et_embedded_kernels.find(kernel_name); From 47aee0fd9de600250fc1a5f790919d9d0ea6672a Mon Sep 17 00:00:00 2001 From: marty1885 Date: Tue, 3 Mar 2026 09:01:15 +0000 Subject: [PATCH 08/12] merge matmul work --- ggml/src/ggml-et/et-kernels/src/platform.h | 81 + ggml/src/ggml-et/ggml-et.cpp | 9 + tests/test-backend-ops.cpp | 10327 +++++++++++++++++++ 3 files changed, 10417 insertions(+) create mode 100644 tests/test-backend-ops.cpp diff --git a/ggml/src/ggml-et/et-kernels/src/platform.h b/ggml/src/ggml-et/et-kernels/src/platform.h index 6825cf9865d6..51af928580d9 100644 --- a/ggml/src/ggml-et/et-kernels/src/platform.h +++ b/ggml/src/ggml-et/et-kernels/src/platform.h @@ -150,6 +150,87 @@ static inline void atomic_store_f16(volatile uint16_t * addr, uint16_t value) { et_global_store_hw(addr, value); } +// Atomic add for F32 values to global memory +// Uses ET hardware's custom amoaddg.w instruction for global atomic add +// This ensures correct accumulation when multiple threads contribute to the same output +static inline void atomic_add_f32(volatile float* addr, float value) { + uint32_t value_bits = *(uint32_t*)&value; + __asm__ volatile( + "amoaddg.w zero, %1, (%0)" + : + : "r"(addr), "r"(value_bits) + : "memory" + ); +} + +// static inline void atomic_add_f32(volatile float* addr, float value) { +// // We use the "f" constraint to ensure 'value' is in a floating-point register (fs1) +// // and "r" for the address in an integer register (rs2) +// __asm__ volatile( +// "famoaddg.pi zero, %1, (%0)" +// : +// : "r"(addr), "f"(value) +// : "memory" +// ); +// } + +// // Atomic Floating Point Swap/Store Global +// static inline void atomic_store_f32(volatile float* addr, float value) { +// uint32_t value_bits = *(uint32_t*)&value; +// __asm__ volatile( +// "famoswapg.pi zero, %1, (%0)" +// : +// : "r"(addr), "f"(value) +// : "memory" +// ); +// } + +// static inline void atomic_add_f32(float *addr, float value) { +// uint32_t old_bits; +// uint32_t expected; +// uint32_t desired; + +// do { +// // Load current value +// __asm__ volatile ( +// "lw %0, 0(%1)" +// : "=r"(old_bits) +// : "r"(addr) +// : "memory" +// ); + +// float old_f; +// __builtin_memcpy(&old_f, &old_bits, sizeof(old_f)); +// float new_f = old_f + value; +// __builtin_memcpy(&desired, &new_f, sizeof(desired)); + +// expected = old_bits; + +// // CAS: rd, expected, desired, (addr) +// __asm__ volatile ( +// "amocmpswapg.w %0, %1, %2, (%3)" +// : "=r"(old_bits) +// : "r"(expected), "r"(desired), "r"(addr) +// : "memory" +// ); + +// } while (old_bits != expected); +// } + + +// static inline void atomic_store_f32(float *addr, float value) { +// uint32_t bits; +// __builtin_memcpy(&bits, &value, sizeof(bits)); + +// __asm__ volatile ( +// "amoswapg.w zero, %1, (%0)" +// : +// : "r"(addr), "r"(bits) +// : "memory" +// ); +// } + + // Atomic store for F16 values to global memory // Uses ET hardware's custom shg instruction (store halfword global) // This ensures cache coherency when multiple threads write to nearby addresses diff --git a/ggml/src/ggml-et/ggml-et.cpp b/ggml/src/ggml-et/ggml-et.cpp index b30209095672..eafaf677350b 100644 --- a/ggml/src/ggml-et/ggml-et.cpp +++ b/ggml/src/ggml-et/ggml-et.cpp @@ -717,6 +717,15 @@ static ggml_status ggml_backend_et_graph_compute(ggml_backend_t backend, ggml_cg case GGML_OP_MUL_MAT: ggml_et_op_mul_mat(dev_ctx, node); + + // if (once < 100){ + // uint64_t * host_data = (uint64_t *) node->data; + + // // printf("Tensor error: %lu\n", host_data[0]); + + // // printf("Tensor error:"); + // once++; + // } break; case GGML_OP_MUL_MAT_ID: diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp new file mode 100644 index 000000000000..084344fb25d7 --- /dev/null +++ b/tests/test-backend-ops.cpp @@ -0,0 +1,10327 @@ +// This file defines tests for various GGML ops and backends. +// For the forward pass it asserts that the results of multiple backends computing the same GGML ops are consistent. +// For the backward pass it asserts that the gradients from backpropagation are consistent +// with the gradients obtained via the method of finite differences ("grad" mode, this is optional). +// It is also possible to check the performance ("perf" mode). +// +// this file has three sections: Section 1 does general setup, section 2 defines the GGML ops to be tested, +// and section 3 defines which tests to run. +// Quick start for adding a new GGML op: Go to section 2 and create a struct that inherits from test_case, +// then go to section 3 and add an instantiation of your struct. + + +// ############################## +// ## Section 1: General Setup ## +// ############################## + + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __EMSCRIPTEN__ +# define N_THREADS 1 +#else +# define N_THREADS std::thread::hardware_concurrency() +#endif + +static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { + size_t nels = ggml_nelements(tensor); + std::vector data(nels); + { + // parallel initialization + static const size_t n_threads = N_THREADS; + + auto init_thread = [&](size_t start, size_t end) { + thread_local std::default_random_engine gen(std::random_device{}()); + std::uniform_real_distribution distribution(min, max); + for (size_t i = start; i < end; i++) { + data[i] = distribution(gen); + } + }; + + if (n_threads == 1) { + init_thread(0, nels); + } else { + std::vector> tasks; + tasks.reserve(n_threads); + for (size_t i = 0; i < n_threads; i++) { + size_t start = i*nels/n_threads; + size_t end = (i+1)*nels/n_threads; + tasks.push_back(std::async(std::launch::async, init_thread, start, end)); + } + for (auto & t : tasks) { + t.get(); + } + } + } + + if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_I32) { + ggml_backend_tensor_set(tensor, data.data(), 0, nels * sizeof(float)); + } else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) { + GGML_ASSERT(nels % ggml_blck_size(tensor->type) == 0); + + // dummy importance matrix + std::vector imatrix(tensor->ne[0], 1.0f); + const float * im = imatrix.data(); + if (!ggml_quantize_requires_imatrix(tensor->type)) { + // when the imatrix is optional, we want to test both quantization with and without imatrix + // use one of the random numbers to decide + if (data[0] > 0.5f*(min + max)) { + im = nullptr; + } + } + + std::vector dataq(ggml_row_size(tensor->type, nels)); + { + // parallel quantization by block + size_t blck_size = ggml_blck_size(tensor->type); + size_t n_blocks = nels / blck_size; + + auto quantize_thread = [&](size_t start, size_t end) { + ggml_quantize_chunk(tensor->type, data.data(), dataq.data(), + start * blck_size, end - start, blck_size, im); + }; + + const size_t min_blocks_per_thread = 1; + const size_t n_quant_threads = std::min(std::max(N_THREADS/2, 1), + std::max(1, n_blocks / min_blocks_per_thread)); + + if (n_quant_threads == 1) { + // single-threaded quantization: do all blocks in the current thread + quantize_thread(0, n_blocks); + } else { + std::vector> tasks; + tasks.reserve(n_quant_threads); + for (size_t i = 0; i < n_quant_threads; i++) { + size_t start = i*n_blocks/n_quant_threads; + size_t end = (i+1)*n_blocks/n_quant_threads; + tasks.push_back(std::async(std::launch::async, quantize_thread, start, end)); + } + for (auto & t : tasks) { + t.get(); + } + } + } + ggml_backend_tensor_set(tensor, dataq.data(), 0, dataq.size()); + } else if (tensor->type == GGML_TYPE_I8 || tensor->type == GGML_TYPE_I16) { + // This is going to create some weird integers though. + ggml_backend_tensor_set(tensor, data.data(), 0, nels * ggml_type_size(tensor->type)); + } else if (tensor->type == GGML_TYPE_I64) { + // Integers with a size of 8 bytes can be set by mirroring the float data, the specific values are again not really meaningful. + const size_t nbytes_half = nels * sizeof(float); + ggml_backend_tensor_set(tensor, data.data(), 0*nbytes_half, nbytes_half); + ggml_backend_tensor_set(tensor, data.data(), 1*nbytes_half, nbytes_half); + } else { + GGML_ABORT("fatal error"); + } +} + +// generate an F16 mask where certain blocks are randomly masked with -INF value +static void init_tensor_kq_mask(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { + GGML_ASSERT(tensor->type == GGML_TYPE_F16); + + GGML_TENSOR_LOCALS( int32_t, ne, tensor, ne); + + std::vector data_f32(ne0*ne1*ne2*ne3); + std::vector data_f16(ne0*ne1*ne2*ne3); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution dis(min, max); + + for (size_t i = 0; i < data_f32.size(); i++) { + data_f32[i] = dis(gen); + } + + // block size + const int blck0 = 128; + const int blck1 = 64; + + // number of INF/zero blocks + const int n_inf_zero_blocks = 0.2*(ne0*ne1*ne2*ne3)/(blck0*blck1); + + for (int b = 0; b < n_inf_zero_blocks; b++) { + const int p3 = (rd() % ne3); + const int p2 = (rd() % ne2); + const int p1 = (rd() % ne1); + const int p0 = (rd() % ne0); + + bool inf = rd() & 1; + + for (int i1 = 0; i1 < blck1 && p1 + i1 < ne1; i1++) { + const int idx = p3*ne2*ne1*ne0 + p2*ne1*ne0 + (p1 + i1)*ne0 + p0; + + for (int i0 = 0; i0 < blck0 && p0 + i0 < ne0; i0++) { + data_f32[idx + i0] = inf ? -INFINITY : 0.0f; + } + } + } + + ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), ne0*ne1*ne2*ne3); + + ggml_backend_tensor_set(tensor, data_f16.data(), 0, data_f16.size()*sizeof(ggml_fp16_t)); +} + +// generate a lower triangular matrix +static void init_tensor_tril(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { + GGML_ASSERT(tensor->type == GGML_TYPE_F32); + GGML_ASSERT(tensor->ne[0] == tensor->ne[1]); + + GGML_TENSOR_LOCALS(int32_t, ne, tensor, ne); + GGML_TENSOR_LOCALS(size_t, nb, tensor, nb); + + std::vector data_f32(ne0*ne1*ne2*ne3); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution dis(min, max); + + for (int64_t i3 = 0; i3 < ne3; i3++) { + for (int64_t i2 = 0; i2 < ne2; i2++) { + for (int64_t i1 = 0; i1 < ne1; i1++) { + for (int64_t i0 = 0; i0 < ne0; i0++) { + int64_t idx = (i0 * nb0 + i1 * nb1 + i2 * nb2 + i3 * nb3) / sizeof(float); + if (i0 <= i1) { + data_f32[idx] = dis(gen); + } else { + data_f32[idx] = 0.0f; + } + } + } + } + } + + ggml_backend_tensor_set(tensor, data_f32.data(), 0, ggml_nbytes(tensor)); +} + +static std::vector tensor_to_float(const ggml_tensor * t) { + std::vector tv; + tv.reserve(ggml_nelements(t)); + + std::vector buf(ggml_nbytes(t)); + ggml_backend_tensor_get(t, buf.data(), 0, ggml_nbytes(t)); + + const auto * tt = ggml_get_type_traits(t->type); + size_t bs = ggml_blck_size(t->type); + std::vector vq(ggml_blck_size(t->type)); + bool quantized = ggml_is_quantized(t->type); + + // access elements by index to avoid gaps in views + for (int64_t i3 = 0; i3 < t->ne[3]; i3++) { + for (int64_t i2 = 0; i2 < t->ne[2]; i2++) { + for (int64_t i1 = 0; i1 < t->ne[1]; i1++) { + for (int64_t i0 = 0; i0 < t->ne[0]; i0 += bs) { + size_t i = i3*t->nb[3] + i2*t->nb[2] + i1*t->nb[1] + i0/bs*t->nb[0]; + if (t->type == GGML_TYPE_F16) { + tv.push_back(ggml_fp16_to_fp32(*(ggml_fp16_t*)&buf[i])); + } else if (t->type == GGML_TYPE_BF16) { + tv.push_back(ggml_bf16_to_fp32(*(ggml_bf16_t*)&buf[i])); + } else if (t->type == GGML_TYPE_F32) { + tv.push_back(*(float *) &buf[i]); + } else if (t->type == GGML_TYPE_I64) { + tv.push_back((float)*(int64_t *) &buf[i]); + } else if (t->type == GGML_TYPE_I32) { + tv.push_back((float)*(int32_t *) &buf[i]); + } else if (t->type == GGML_TYPE_I16) { + tv.push_back((float)*(int16_t *) &buf[i]); + } else if (t->type == GGML_TYPE_I8) { + tv.push_back((float)*(int8_t *) &buf[i]); + } else if (quantized) { + tt->to_float(&buf[i], vq.data(), bs); + tv.insert(tv.end(), vq.begin(), vq.end()); + } else { + GGML_ABORT("fatal error"); + } + } + } + } + } + + return tv; +} + +// normalized mean squared error = mse(a, b) / mse(a, 0) +static double nmse(const float * a, const float * b, size_t n) { + double mse_a_b = 0.0; + double mse_a_0 = 0.0; + + for (size_t i = 0; i < n; i++) { + float a_i = a[i]; + float b_i = b[i]; + + mse_a_b += (a_i - b_i) * (a_i - b_i); + mse_a_0 += a_i * a_i; + } + + return mse_a_b / mse_a_0; +} + +// difference between 2 sets (Jaccard distance, 0 - no difference, 1 - no overlap) +template +static double jdst(const T * a, const T * b, size_t n) { + std::unordered_map set_a; + std::unordered_map set_b; + + for (size_t i = 0; i < n; ++i) { + set_a[a[i]]++; + set_b[b[i]]++; + } + + size_t diff = 0; + + for (const auto & p : set_a) { + const int64_t na = p.second; + const int64_t nb = set_b.find(p.first) != set_b.end() ? set_b.at(p.first) : 0; + + diff += std::abs(na - nb); + } + + for (const auto & p : set_b) { + if (set_a.find(p.first) == set_a.end()) { + diff += p.second; + } + } + + return (double) diff / (2*n); +} + +// maximum absolute asymmetry between a and b +// asymmetry: (a - b) / (a + b) +// This is more stable than relative error if one of the values fluctuates towards zero. +// n: number of values to compare. +// expected_vals: optional vector of expected values for a. If expected_vals is not empty, filter out all comparisons where +// a does not match any of the expected values. Needed for noncontinuous gradients where the numerical calculation can fail. +static double mean_abs_asymm(const float * a, const float * b, const size_t n, const std::vector & expected_vals) { + double sum = 0.0f; + + size_t nvalid = 0; + for (size_t i = 0; i < n; i++) { + if (!expected_vals.empty()) { + bool matches_any = false; + for (const float & ev : expected_vals) { + if (fabsf(a[i] - ev) < 1e-3f) { + matches_any = true; + break; + } + } + if (!matches_any) { + continue; + } + } + + const float asymm = (a[i] - b[i]) / (a[i] + b[i]); + + sum += fabsf(asymm); + nvalid++; + } + + return sum/nvalid; +} + +// utils for printing the variables of the test cases + +static std::string var_to_str(const std::string & x) { + return x; +} + +template +static std::string var_to_str(const T & x) { + return std::to_string(x); +} + +template +static std::string var_to_str(const T (&x)[N]) { + std::string s = "["; + for (size_t i = 0; i < N; i++) { + if (i > 0) { + s += ","; + } + s += var_to_str(x[i]); + } + s += "]"; + return s; +} + +template +static std::string var_to_str(const std::array & x) { + std::string s = "["; + for (size_t i = 0; i < N; i++) { + if (i > 0) { + s += ","; + } + s += var_to_str(x[i]); + } + s += "]"; + return s; +} + +static std::string var_to_str(ggml_type type) { + return ggml_type_name(type); +} + +static std::string var_to_str(ggml_prec prec) { + return prec == GGML_PREC_F32 ? "f32" : "def"; +} + +static std::string var_to_str(ggml_op_pool pool) { + switch (pool) { + case GGML_OP_POOL_AVG: return "avg"; + case GGML_OP_POOL_MAX: return "max"; + default: return std::to_string(pool); + } +} + +static std::string var_to_str(ggml_scale_mode mode) { + std::string str; + switch (mode & 0xFF) { + case GGML_SCALE_MODE_NEAREST: str = "nearest"; break; + case GGML_SCALE_MODE_BILINEAR: str = "bilinear"; break; + case GGML_SCALE_MODE_BICUBIC: str = "bicubic"; break; + default: str = std::to_string(mode); break; + } + if (mode & GGML_SCALE_FLAG_ALIGN_CORNERS) { + str += "|align_corners"; + } + if (mode & GGML_SCALE_FLAG_ANTIALIAS) { + str += "|antialias"; + } + return str; +} + +#define VAR_TO_STR(x) (#x "=" + var_to_str(x)) + +#define VARS_TO_STR1(a) VAR_TO_STR(a) +#define VARS_TO_STR2(a, b) VAR_TO_STR(a) + "," + VAR_TO_STR(b) +#define VARS_TO_STR3(a, b, c) VAR_TO_STR(a) + "," + VARS_TO_STR2(b, c) +#define VARS_TO_STR4(a, b, c, d) VAR_TO_STR(a) + "," + VARS_TO_STR3(b, c, d) +#define VARS_TO_STR5(a, b, c, d, e) VAR_TO_STR(a) + "," + VARS_TO_STR4(b, c, d, e) +#define VARS_TO_STR6(a, b, c, d, e, f) VAR_TO_STR(a) + "," + VARS_TO_STR5(b, c, d, e, f) +#define VARS_TO_STR7(a, b, c, d, e, f, g) VAR_TO_STR(a) + "," + VARS_TO_STR6(b, c, d, e, f, g) +#define VARS_TO_STR8(a, b, c, d, e, f, g, h) VAR_TO_STR(a) + "," + VARS_TO_STR7(b, c, d, e, f, g, h) +#define VARS_TO_STR9(a, b, c, d, e, f, g, h, i) VAR_TO_STR(a) + "," + VARS_TO_STR8(b, c, d, e, f, g, h, i) +#define VARS_TO_STR10(a, b, c, d, e, f, g, h, i, j) VAR_TO_STR(a) + "," + VARS_TO_STR9(b, c, d, e, f, g, h, i, j) +#define VARS_TO_STR11(a, b, c, d, e, f, g, h, i, j, k) VAR_TO_STR(a) + "," + VARS_TO_STR10(b, c, d, e, f, g, h, i, j, k) +#define VARS_TO_STR12(a, b, c, d, e, f, g, h, i, j, k, l) VAR_TO_STR(a) + "," + VARS_TO_STR11(b, c, d, e, f, g, h, i, j, k, l) +#define VARS_TO_STR13(a, b, c, d, e, f, g, h, i, j, k, l, m) VAR_TO_STR(a) + "," + VARS_TO_STR12(b, c, d, e, f, g, h, i, j, k, l, m) +#define VARS_TO_STR14(a, b, c, d, e, f, g, h, i, j, k, l, m, n) VAR_TO_STR(a) + "," + VARS_TO_STR13(b, c, d, e, f, g, h, i, j, k, l, m, n) +#define VARS_TO_STR15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) VAR_TO_STR(a) + "," + VARS_TO_STR14(b, c, d, e, f, g, h, i, j, k, l, m, n, o) +#define VARS_TO_STR16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) VAR_TO_STR(a) + "," + VARS_TO_STR15(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) + +#ifdef GGML_USE_SYCL +static bool inline _isinf(float f) { + return (*(uint32_t *)&f & 0x7fffffff) == 0x7f800000; +} +#else +static bool inline _isinf(float f) { return std::isinf(f); } +#endif + +// accept FLT_MAX as infinity +static bool isinf_or_max(float f) { + return _isinf(f) || f == FLT_MAX || f == -FLT_MAX; +} + +static bool ggml_is_view_op(enum ggml_op op) { + return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE; +} + +static bool backend_has_feature(ggml_backend_t backend, const char * feature_name) { + ggml_backend_dev_t dev = ggml_backend_get_device(backend); + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); + + auto get_features = (ggml_backend_get_features_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_get_features"); + if (!get_features) { + return false; + } + + const ggml_backend_feature * features = get_features(reg); + if (!features) { + return false; + } + + for (const ggml_backend_feature * f = features; f->name; ++f) { + if (strcmp(f->name, feature_name) == 0 && strcmp(f->value, "1") == 0) { + return true; + } + } + return false; +} + +enum test_mode { + MODE_TEST, + MODE_PERF, + MODE_GRAD, + MODE_SUPPORT, +}; + +// Output format support similar to llama-bench +enum output_formats { CONSOLE, SQL, CSV }; + +static const char * output_format_str(output_formats format) { + switch (format) { + case CONSOLE: + return "console"; + case SQL: + return "sql"; + case CSV: + return "csv"; + default: + GGML_ABORT("invalid output format"); + } +} + +static bool output_format_from_str(const std::string & s, output_formats & format) { + if (s == "console") { + format = CONSOLE; + } else if (s == "sql") { + format = SQL; + } else if (s == "csv") { + format = CSV; + } else { + return false; + } + return true; +} + +static std::string test_time_now() { + time_t t = time(NULL); + struct tm tm_buf; +#ifdef _WIN32 + if (gmtime_s(&tm_buf, &t) != 0) { + return ""; + } +#else + if (gmtime_r(&t, &tm_buf) == nullptr) { + return ""; + } +#endif + char buf[32]; + if (std::strftime(buf, sizeof(buf), "%FT%TZ", &tm_buf) == 0) { + return ""; + } + return buf; +} + +// Test result structure for SQL output +struct test_result { + std::string test_time; + std::string build_commit; + std::string backend_name; + std::string op_name; + std::string op_params; + std::string test_mode; + bool supported; + bool passed; + std::string error_message; + double time_us; + double flops; + double bandwidth_gb_s; + size_t memory_kb; + int n_runs; + std::string device_description; + std::string backend_reg_name; + + test_result() { + // Initialize with default values + time_us = 0.0; + flops = 0.0; + bandwidth_gb_s = 0.0; + memory_kb = 0; + n_runs = 0; + supported = false; + passed = false; + + test_time = test_time_now(); + + // Set build info + build_commit = ggml_commit(); + } + + test_result(const std::string & backend_name, const std::string & op_name, const std::string & op_params, + const std::string & test_mode, bool supported, bool passed, const std::string & error_message = "", + double time_us = 0.0, double flops = 0.0, double bandwidth_gb_s = 0.0, size_t memory_kb = 0, + int n_runs = 0, const std::string & device_description = "", const std::string & backend_reg_name = "") : + backend_name(backend_name), + op_name(op_name), + op_params(op_params), + test_mode(test_mode), + supported(supported), + passed(passed), + error_message(error_message), + time_us(time_us), + flops(flops), + bandwidth_gb_s(bandwidth_gb_s), + memory_kb(memory_kb), + n_runs(n_runs), + device_description(device_description), + backend_reg_name(backend_reg_name) { + test_time = test_time_now(); + + // Set build info + build_commit = ggml_commit(); + } + + static const std::vector & get_fields() { + static const std::vector fields = { + "test_time", "build_commit", "backend_name", "op_name", "op_params", "test_mode", "supported", + "passed", "error_message", "time_us", "flops", "bandwidth_gb_s", "memory_kb", "n_runs", + "device_description", "backend_reg_name" + }; + return fields; + } + + enum field_type { STRING, BOOL, INT, FLOAT }; + + static field_type get_field_type(const std::string & field) { + if (field == "supported" || field == "passed") { + return BOOL; + } + if (field == "memory_kb" || field == "n_runs") { + return INT; + } + if (field == "time_us" || field == "flops" || field == "bandwidth_gb_s") { + return FLOAT; + } + return STRING; + } + + std::vector get_values() const { + return { test_time, + build_commit, + backend_name, + op_name, + op_params, + test_mode, + std::to_string(supported), + std::to_string(passed), + error_message, + std::to_string(time_us), + std::to_string(flops), + std::to_string(bandwidth_gb_s), + std::to_string(memory_kb), + std::to_string(n_runs), + device_description, + backend_reg_name }; + } +}; + +// Printer classes for different output formats +enum class test_status_t { NOT_SUPPORTED, OK, FAIL, SKIPPED }; + +struct test_operation_info { + std::string op_name; + std::string op_params; + std::string backend_name; + test_status_t status = test_status_t::OK; + std::string failure_reason; + + // Additional information fields that were previously in separate structs + std::string error_component; + std::string error_details; + + // Gradient info + int64_t gradient_index = -1; + std::string gradient_param_name; + float gradient_value = 0.0f; + + // MAA error info + double maa_error = 0.0; + double maa_threshold = 0.0; + + // Flags for different types of information + bool has_error = false; + bool has_gradient_info = false; + bool has_maa_error = false; + bool is_compare_failure = false; + bool is_large_tensor_skip = false; + + test_operation_info() = default; + + test_operation_info(const std::string & op_name, const std::string & op_params, const std::string & backend_name, + test_status_t status = test_status_t::OK, const std::string & failure_reason = "") : + op_name(op_name), + op_params(op_params), + backend_name(backend_name), + status(status), + failure_reason(failure_reason) {} + + // Set error information + void set_error(const std::string & component, const std::string & details) { + has_error = true; + error_component = component; + error_details = details; + if (status == test_status_t::OK) { + status = test_status_t::FAIL; + } + } + + // Set gradient information + void set_gradient_info(int64_t index, const std::string & param_name, float value) { + has_gradient_info = true; + gradient_index = index; + gradient_param_name = param_name; + gradient_value = value; + if (status == test_status_t::OK) { + status = test_status_t::FAIL; + } + } + + // Set MAA error information + void set_maa_error(double error, double threshold) { + has_maa_error = true; + maa_error = error; + maa_threshold = threshold; + if (status == test_status_t::OK) { + status = test_status_t::FAIL; + } + } + + // Set compare failure + void set_compare_failure() { + is_compare_failure = true; + if (status == test_status_t::OK) { + status = test_status_t::FAIL; + } + } + + // Set large tensor skip + void set_large_tensor_skip() { is_large_tensor_skip = true; } +}; + +struct test_summary_info { + size_t tests_passed; + size_t tests_total; + bool is_backend_summary = false; // true for backend summary, false for test summary + + test_summary_info() = default; + + test_summary_info(size_t tests_passed, size_t tests_total, bool is_backend_summary = false) : + tests_passed(tests_passed), + tests_total(tests_total), + is_backend_summary(is_backend_summary) {} +}; + +struct testing_start_info { + size_t device_count; + + testing_start_info() = default; + + testing_start_info(size_t device_count) : device_count(device_count) {} +}; + +struct backend_init_info { + size_t device_index; + size_t total_devices; + std::string device_name; + bool skipped = false; + std::string skip_reason; + std::string description; + size_t memory_total_mb = 0; + size_t memory_free_mb = 0; + bool has_memory_info = false; + + backend_init_info() = default; + + backend_init_info(size_t device_index, size_t total_devices, const std::string & device_name, bool skipped = false, + const std::string & skip_reason = "", const std::string & description = "", + size_t memory_total_mb = 0, size_t memory_free_mb = 0, bool has_memory_info = false) : + device_index(device_index), + total_devices(total_devices), + device_name(device_name), + skipped(skipped), + skip_reason(skip_reason), + description(description), + memory_total_mb(memory_total_mb), + memory_free_mb(memory_free_mb), + has_memory_info(has_memory_info) {} +}; + +struct backend_status_info { + std::string backend_name; + test_status_t status; + + backend_status_info() = default; + + backend_status_info(const std::string & backend_name, test_status_t status) : + backend_name(backend_name), + status(status) {} +}; + +struct overall_summary_info { + size_t backends_passed; + size_t backends_total; + bool all_passed; + + overall_summary_info() = default; + + overall_summary_info(size_t backends_passed, size_t backends_total, bool all_passed) : + backends_passed(backends_passed), + backends_total(backends_total), + all_passed(all_passed) {} +}; + +struct printer { + virtual ~printer() {} + + FILE * fout = stdout; + + virtual void print_header() {} + + virtual void print_test_result(const test_result & result) = 0; + + virtual void print_footer() {} + + virtual void print_operation(const test_operation_info & info) { (void) info; } + + virtual void print_summary(const test_summary_info & info) { (void) info; } + + virtual void print_testing_start(const testing_start_info & info) { (void) info; } + + virtual void print_backend_init(const backend_init_info & info) { (void) info; } + + virtual void print_backend_status(const backend_status_info & info) { (void) info; } + + virtual void print_overall_summary(const overall_summary_info & info) { (void) info; } + + virtual void print_failed_tests(const std::vector & failed_tests) { (void) failed_tests; } +}; + +struct console_printer : public printer { + void print_test_result(const test_result & result) override { + if (result.test_mode == "test") { + print_test_console(result); + } else if (result.test_mode == "perf") { + print_perf_console(result); + } else if (result.test_mode == "support") { + print_support_console(result); + } + } + + void print_operation(const test_operation_info & info) override { + printf(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str()); + fflush(stdout); + + // Handle large tensor skip first + if (info.is_large_tensor_skip) { + printf("skipping large tensors for speed \n"); + return; + } + + // Handle not supported status + if (info.status == test_status_t::NOT_SUPPORTED) { + if (!info.failure_reason.empty()) { + printf("not supported [%s]\n", info.failure_reason.c_str()); + } else { + printf("not supported [%s]\n", info.backend_name.c_str()); + } + return; + } + + // Handle errors and additional information + if (info.has_error) { + if (info.error_component == "allocation") { + fprintf(stderr, "failed to allocate tensors [%s] ", info.backend_name.c_str()); + } else if (info.error_component == "backend") { + fprintf(stderr, " Failed to initialize %s backend\n", info.backend_name.c_str()); + } else { + fprintf(stderr, "Error in %s: %s\n", info.error_component.c_str(), info.error_details.c_str()); + } + } + + // Handle gradient info + if (info.has_gradient_info) { + printf("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", info.op_name.c_str(), info.gradient_index, + info.gradient_param_name.c_str(), info.gradient_value); + } + + // Handle MAA error + if (info.has_maa_error) { + printf("[%s] MAA = %.9f > %.9f ", info.op_name.c_str(), info.maa_error, info.maa_threshold); + } + + // Handle compare failure + if (info.is_compare_failure) { + printf("compare failed "); + } + + // Print final status + if (info.status == test_status_t::OK) { + printf("\033[1;32mOK\033[0m\n"); + } else { + printf("\033[1;31mFAIL\033[0m\n"); + } + } + + void print_summary(const test_summary_info & info) override { + if (info.is_backend_summary) { + printf("%zu/%zu backends passed\n", info.tests_passed, info.tests_total); + } else { + printf(" %zu/%zu tests passed\n", info.tests_passed, info.tests_total); + } + } + + void print_backend_status(const backend_status_info & info) override { + printf(" Backend %s: ", info.backend_name.c_str()); + if (info.status == test_status_t::OK) { + printf("\033[1;32mOK\033[0m\n"); + } else { + printf("\033[1;31mFAIL\033[0m\n"); + } + } + + void print_testing_start(const testing_start_info & info) override { + printf("Testing %zu devices\n\n", info.device_count); + } + + void print_backend_init(const backend_init_info & info) override { + printf("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str()); + + if (info.skipped) { + printf(" %s\n", info.skip_reason.c_str()); + return; + } + + if (!info.description.empty()) { + printf(" Device description: %s\n", info.description.c_str()); + } + + if (info.has_memory_info) { + printf(" Device memory: %zu MB (%zu MB free)\n", info.memory_total_mb, info.memory_free_mb); + } + + printf("\n"); + } + + void print_overall_summary(const overall_summary_info & info) override { + printf("%zu/%zu backends passed\n", info.backends_passed, info.backends_total); + if (info.all_passed) { + printf("\033[1;32mOK\033[0m\n"); + } else { + printf("\033[1;31mFAIL\033[0m\n"); + } + } + + void print_failed_tests(const std::vector & failed_tests) override { + if (failed_tests.empty()) { + return; + } + + printf("\nFailing tests:\n"); + for (const auto & test_name : failed_tests) { + printf(" %s\n", test_name.c_str()); + } + } + + private: + void print_test_console(const test_result & result) { + printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); + fflush(stdout); + + if (!result.supported) { + printf("not supported [%s] ", result.backend_name.c_str()); + printf("\n"); + return; + } + + if (result.passed) { + printf("\033[1;32mOK\033[0m\n"); + } else { + printf("\033[1;31mFAIL\033[0m\n"); + } + } + + void print_perf_console(const test_result & result) { + int len = printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); + fflush(stdout); + + if (!result.supported) { + printf("not supported\n"); + return; + } + + // align while also leaving some margin for variations in parameters + int align = 8; + int last = (len + align - 1) / align * align; + if (last - len < 5) { + last += align; + } + printf("%*s", last - len, ""); + + printf(" %8d runs - %8.2f us/run - ", result.n_runs, result.time_us); + + if (result.flops > 0) { + auto format_flops = [](double flops) -> std::string { + char buf[256]; + if (flops >= 1e12) { + snprintf(buf, sizeof(buf), "%6.2f TFLOP", flops / 1e12); + } else if (flops >= 1e9) { + snprintf(buf, sizeof(buf), "%6.2f GFLOP", flops / 1e9); + } else if (flops >= 1e6) { + snprintf(buf, sizeof(buf), "%6.2f MFLOP", flops / 1e6); + } else { + snprintf(buf, sizeof(buf), "%6.2f kFLOP", flops / 1e3); + } + return buf; + }; + uint64_t op_flops_per_run = result.flops * result.time_us / 1e6; + printf("%s/run - \033[1;34m%sS\033[0m", format_flops(op_flops_per_run).c_str(), + format_flops(result.flops).c_str()); + } else { + printf("%8zu kB/run - \033[1;34m%7.2f GB/s\033[0m", result.memory_kb, result.bandwidth_gb_s); + } + printf("\n"); + } + + void print_support_console(const test_result & result) { + printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); + fflush(stdout); + + if (result.supported) { + printf("\033[1;32mSUPPORTED\033[0m\n"); + } else { + printf("\033[1;31mNOT SUPPORTED\033[0m\n"); + } + } +}; + +struct sql_printer : public printer { + static std::string get_sql_field_type(const std::string & field) { + switch (test_result::get_field_type(field)) { + case test_result::STRING: + return "TEXT"; + case test_result::BOOL: + case test_result::INT: + return "INTEGER"; + case test_result::FLOAT: + return "REAL"; + default: + GGML_ABORT("invalid field type"); + } + } + + void print_header() override { + std::vector fields = test_result::get_fields(); + fprintf(fout, "CREATE TABLE IF NOT EXISTS test_backend_ops (\n"); + for (size_t i = 0; i < fields.size(); i++) { + fprintf(fout, " %s %s%s\n", fields[i].c_str(), get_sql_field_type(fields[i]).c_str(), + i < fields.size() - 1 ? "," : ""); + } + fprintf(fout, ");\n\n"); + } + + void print_test_result(const test_result & result) override { + fprintf(fout, "INSERT INTO test_backend_ops ("); + std::vector fields = test_result::get_fields(); + for (size_t i = 0; i < fields.size(); i++) { + fprintf(fout, "%s%s", fields[i].c_str(), i < fields.size() - 1 ? ", " : ""); + } + fprintf(fout, ") VALUES ("); + std::vector values = result.get_values(); + for (size_t i = 0; i < values.size(); i++) { + fprintf(fout, "'%s'%s", values[i].c_str(), i < values.size() - 1 ? ", " : ""); + } + fprintf(fout, ");\n"); + } +}; + +struct csv_printer : public printer { + void print_header() override { + + std::vector fields = test_result::get_fields(); + std::vector fields_csv = get_fields_csv(); + for (size_t i = 0; i < fields.size(); i++) { + if (std::find(std::begin(fields_csv), std::end(fields_csv), fields[i]) == std::end(fields_csv)) { + continue; + } + printf("\"%s\"%s", fields[i].c_str(), i < fields.size() - 1 ? "," : ""); + } + printf("\n"); + } + + void print_test_result(const test_result & result) override { + + std::vector values = result.get_values(); + std::vector fields = test_result::get_fields(); + std::vector fields_csv = get_fields_csv(); + + for (size_t i = 0; i < values.size(); i++) { + + if (std::find(std::begin(fields_csv), std::end(fields_csv), fields[i]) == std::end(fields_csv)) { + continue; + } + + // Escape quotes and wrap in quotes for CSV + std::string escaped_value = values[i]; + size_t pos = 0; + while ((pos = escaped_value.find("\"", pos)) != std::string::npos) { + escaped_value.replace(pos, 1, "\"\""); + pos += 2; + } + printf("\"%s\"%s", escaped_value.c_str(), i < values.size() - 1 ? "," : ""); + } + printf("\n"); + } + + static std::vector get_fields_csv() { + return { + "op_name", + "op_params", + "supported", + "error_message", + "test_mode", + "backend_reg_name", + "backend_name", + }; + } + +}; + +static std::unique_ptr create_printer(output_formats format) { + switch (format) { + case CONSOLE: + return std::make_unique(); + case SQL: + return std::make_unique(); + case CSV: + return std::make_unique(); + } + GGML_ABORT("invalid output format"); +} + +static std::mutex g_test_output_mutex; + +static void print_test_result_locked(printer * output_printer, const test_result & result) { + if (output_printer == nullptr) { + return; + } + + std::lock_guard guard(g_test_output_mutex); + output_printer->print_test_result(result); +} + +struct test_case { + virtual ~test_case() {} + + virtual std::string op_desc(ggml_tensor * t) { + return ggml_op_desc(t); + } + + virtual std::string vars() { + return ""; + } + + virtual ggml_tensor * build_graph(ggml_context * ctx) = 0; + virtual ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) { + GGML_UNUSED(ctx_weights); + return build_graph(ctx); + } + + virtual double max_nmse_err() { + return 1e-7; + } + + virtual double max_nmse_err(ggml_backend_t backend) { + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); + // See https://github.com/ggml-org/llama.cpp/pull/22976 for explanation. + if (contains_f16 && strcmp(ggml_backend_reg_name(reg), "WebGPU") == 0) { + return std::max(max_nmse_err(), 1e-6); + } + return max_nmse_err(); + } + + virtual double max_maa_err() { + return 1e-4; + } + + virtual double max_err() { + return max_nmse_err(); + } + + virtual double max_err(ggml_backend_t backend) { + return max_nmse_err(backend); + } + + virtual double err(const float * a, const float * b, size_t n) { + return nmse(a, b, n); + } + + virtual float grad_eps() { + return 1e-1f; + } + + // If false, estimate gradient with 2 points, neglects 3rd order derivative and higher. + // If true, estimate gradient with 4 points, neglects 5th order derivative and higher. + virtual bool grad_precise() { + return false; + } + + // Skip gradient checks if total number of gradients to be checked is larger than this (to speed up the tests). + virtual int64_t grad_nmax() { + return 10000; + } + + // No effect if empty. + // If not empty, skip all gradient checks where the numerical result does not match any of the values. + // Needed for dealing with noncontinuous gradients (e.g. ReLU) where estimation using finite differences is unreliable. + virtual std::vector grad_expect() { + return {}; + } + + virtual void initialize_tensors(ggml_context * ctx) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t); + } + } + + virtual size_t op_size(ggml_tensor * t) { + size_t size = ggml_nbytes(t); + // add source tensors + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (t->src[i] != NULL) { + size += ggml_nbytes(t->src[i]); + } + } + return size; + } + + virtual uint64_t op_flops(ggml_tensor * t) { + GGML_UNUSED(t); + return 0; + } + + virtual bool run_whole_graph() { return false; } + virtual std::vector fusion_test_nodes() { return {}; } + virtual bool use_weight_context() { return false; } + + ggml_cgraph * gf = nullptr; + ggml_cgraph * gb = nullptr; + + static const int sentinel_size = 1024; + + test_mode mode; + + std::vector sentinels; + + std::string current_op_name; + bool contains_f16 = false; + + // Used by the WebGPU backend to relax error thresholds on ops on f16 tensors + void check_for_f16_tensor(ggml_context * ctx) { + contains_f16 = false; + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_F16) { + contains_f16 = true; + break; + } + } + } + + void add_sentinel(ggml_context * ctx) { + if (mode == MODE_PERF || mode == MODE_GRAD || mode == MODE_SUPPORT) { + return; + } + ggml_tensor * sentinel = ::ggml_new_tensor_1d(ctx, GGML_TYPE_F32, sentinel_size); + ggml_format_name(sentinel, "sent_%zu", sentinels.size()); + sentinels.push_back(sentinel); + } + + // hijack ggml_new_tensor to add sentinels after each tensor to check for overflows in the backend + + ggml_tensor * ggml_new_tensor(ggml_context * ctx, ggml_type type, int n_dims, const int64_t * ne) { + ggml_tensor * t = ::ggml_new_tensor(ctx, type, n_dims, ne); + add_sentinel(ctx); + return t; + } + + ggml_tensor * ggml_new_tensor_1d(ggml_context * ctx, ggml_type type, int64_t ne0) { + ggml_tensor * t = ::ggml_new_tensor_1d(ctx, type, ne0); + add_sentinel(ctx); + return t; + } + + ggml_tensor * ggml_new_tensor_2d(ggml_context * ctx, ggml_type type, int64_t ne0, int64_t ne1) { + ggml_tensor * t = ::ggml_new_tensor_2d(ctx, type, ne0, ne1); + add_sentinel(ctx); + return t; + } + + ggml_tensor * ggml_new_tensor_3d(ggml_context * ctx, ggml_type type, int64_t ne0, int64_t ne1, int64_t ne2) { + ggml_tensor * t = ::ggml_new_tensor_3d(ctx, type, ne0, ne1, ne2); + add_sentinel(ctx); + return t; + } + + ggml_tensor * ggml_new_tensor_4d(ggml_context * ctx, ggml_type type, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3) { + ggml_tensor * t = ::ggml_new_tensor_4d(ctx, type, ne0, ne1, ne2, ne3); + add_sentinel(ctx); + return t; + } + + // Checks an op against the test filter, which is a comma separated list of OP names or specific variations + bool matches_filter(ggml_tensor * op, const char * op_names_filter) { + if (op_names_filter) { + const auto op_name = op_desc(op); + const auto op_full_name = op_name + "(" + vars() + ")"; + std::string_view filter(op_names_filter); + while (!filter.empty()) { + auto comma_pos = filter.find_first_of(','); + const auto lparen_pos = filter.find_first_of('('); + if (lparen_pos < comma_pos) { + auto rparen_pos = filter.find_first_of(')'); + comma_pos = filter.find_first_of(',', rparen_pos); + const auto op_filter = filter.substr(0, comma_pos); + if (op_filter == op_full_name) { + return true; + } + } else { + const auto op_filter = filter.substr(0, comma_pos); + if (op_filter == op_name) { + return true; + } + } + filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : ""; + } + return false; + } else { + return true; + } + } + + test_status_t eval(ggml_backend_t backend1, + ggml_backend_t backend2, + const char * op_names_filter, + printer * output_printer) { + mode = MODE_TEST; + + ggml_init_params params = { + /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead(), + /* .mem_base = */ NULL, + /* .no_alloc = */ true, + }; + const bool use_weights = use_weight_context(); + + ggml_context_ptr ctx(ggml_init(params)); + GGML_ASSERT(ctx); + ggml_context_ptr ctx_weights(use_weights ? ggml_init(params) : nullptr); + GGML_ASSERT(!use_weights || ctx_weights); + + gf = ggml_new_graph(ctx.get()); + + // pre-graph sentinel + add_sentinel(ctx.get()); + if (ctx_weights) { + add_sentinel(ctx_weights.get()); + } + + ggml_tensor * out = build_graph(ctx.get(), ctx_weights.get()); + current_op_name = op_desc(out); + check_for_f16_tensor(ctx.get()); + + if (!matches_filter(out, op_names_filter)) { + //printf(" %s: skipping\n", op_desc(out).c_str()); + return test_status_t::SKIPPED; + } + + // check if the backends support the ops + bool supported = true; + for (ggml_backend_t backend : {backend1, backend2}) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { + if (!ggml_backend_supports_op(backend, t)) { + supported = false; + break; + } + } + } + + if (!supported) { + // Create test result for unsupported operation + test_result result(ggml_backend_name(backend1), current_op_name, vars(), "test", + false, false, "not supported"); + + print_test_result_locked(output_printer, result); + + return test_status_t::NOT_SUPPORTED; + } + + // post-graph sentinel + add_sentinel(ctx.get()); + if (ctx_weights) { + add_sentinel(ctx_weights.get()); + } + + ggml_backend_buffer_ptr buf_weights(nullptr); + if (ctx_weights) { + buf_weights.reset(ggml_backend_alloc_ctx_tensors(ctx_weights.get(), backend1)); + if (buf_weights == NULL) { + printf("failed to allocate weight tensors [%s] ", ggml_backend_name(backend1)); + return test_status_t::FAIL; + } + ggml_backend_buffer_set_usage(buf_weights.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + } + + // allocate + ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend1)); + + if (buf == NULL) { + printf("failed to allocate tensors [%s] ", ggml_backend_name(backend1)); + return test_status_t::FAIL; + } + + // build graph + ggml_build_forward_expand(gf, out); + + // add sentinels as graph nodes so that they are checked in the callback + for (ggml_tensor * sentinel : sentinels) { + ggml_graph_add_node(gf, sentinel); + } + + // randomize tensors + initialize_tensors(ctx.get()); + if (ctx_weights) { + initialize_tensors(ctx_weights.get()); + } + + // compare + struct callback_userdata { + bool ok; + test_case * tc; + ggml_backend_t backend1; + ggml_backend_t backend2; + }; + + callback_userdata ud { + true, + this, + backend1, + backend2, + }; + + auto callback = [](int index, ggml_tensor * t1, ggml_tensor * t2, void * user_data) -> bool { + callback_userdata * ud = (callback_userdata *) user_data; + const char * bn1 = ggml_backend_name(ud->backend1); + const char * bn2 = ggml_backend_name(ud->backend2); + + if (t1->op == GGML_OP_NONE) { + // sentinels must be unchanged + std::vector t1_data(ggml_nbytes(t1)); + std::vector t2_data(ggml_nbytes(t2)); + ggml_backend_tensor_get(t1, t1_data.data(), 0, ggml_nbytes(t1)); + ggml_backend_tensor_get(t2, t2_data.data(), 0, ggml_nbytes(t2)); + + if (memcmp(t1_data.data(), t2_data.data(), ggml_nbytes(t1)) != 0) { + printf("sentinel mismatch: %s ", t1->name); + ud->ok = false; + return true; + } + } + + std::vector f1 = tensor_to_float(t1); + std::vector f2 = tensor_to_float(t2); + + for (size_t i = 0; i < f1.size(); i++) { + // check for nans + if (std::isnan(f1[i]) || std::isnan(f2[i])) { + printf("[%s] NaN at index %zu (%s=%f %s=%f) ", ggml_op_desc(t1), i, bn1, f1[i], bn2, f2[i]); + ud->ok = false; + return true; + } + // check for infs: both must be inf of the same sign, or both must be finite + if (isinf_or_max(f1[i]) || isinf_or_max(f2[i])) { + if (isinf_or_max(f1[i]) && isinf_or_max(f2[i])) { + if (std::signbit(f1[i]) != std::signbit(f2[i])) { + printf("[%s] inf sign mismatch: %s=%f %s=%f ", ggml_op_desc(t1), bn1, f1[i], bn2, f2[i]); + ud->ok = false; + return true; + } + } else { + printf("[%s] inf mismatch: %s=%f %s=%f ", ggml_op_desc(t1), bn1, f1[i], bn2, f2[i]); + ud->ok = false; + return true; + } + } + } + + double err = ud->tc->err(f1.data(), f2.data(), f1.size()); + if (err > ud->tc->max_err(ud->backend1)) { + printf("[%s] ERR = %.9f > %.9f ", ggml_op_desc(t1), err, ud->tc->max_err(ud->backend1)); + //for (int i = 0; i < (int) f1.size(); i++) { + // printf("%5d %9.6f %9.6f, diff = %9.6f\n", i, f1[i], f2[i], f1[i] - f2[i]); + //} + //printf("\n"); + //exit(1); + ud->ok = false; + } + return true; + + GGML_UNUSED(index); + }; + + std::vector fused_nodes_to_verify = fusion_test_nodes(); + if (fused_nodes_to_verify.size() == 0 && run_whole_graph()) { + fused_nodes_to_verify.push_back(out); + } + const bool cmp_ok = ggml_backend_compare_graph_backend(backend1, backend2, gf, callback, &ud, + run_whole_graph() ? fused_nodes_to_verify.data() : nullptr, + fused_nodes_to_verify.size()); + + // Create test result + bool test_passed = ud.ok && cmp_ok; + std::string error_msg = test_passed ? "" : (!cmp_ok ? "compare failed" : "test failed"); + test_result result(ggml_backend_name(backend1), current_op_name, vars(), "test", supported, test_passed, + error_msg); + + print_test_result_locked(output_printer, result); + + return test_passed ? test_status_t::OK : test_status_t::FAIL; + } + + bool eval_perf(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) { + mode = MODE_PERF; + + static const size_t graph_nodes = 8192; + + ggml_init_params params = { + /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead_custom(graph_nodes, false), + /* .mem_base = */ NULL, + /* .no_alloc = */ true, + }; + const bool use_weights = use_weight_context(); + + ggml_context_ptr ctx(ggml_init(params)); // smart ptr + GGML_ASSERT(ctx); + ggml_context_ptr ctx_weights(use_weights ? ggml_init(params) : nullptr); + GGML_ASSERT(!use_weights || ctx_weights); + + ggml_tensor * out = build_graph(ctx.get(), ctx_weights.get()); + current_op_name = op_desc(out); + if (!matches_filter(out, op_names_filter)) { + //printf(" %s: skipping\n", op_desc(out).c_str()); + return true; + } + + if (!ggml_backend_supports_op(backend, out)) { + // Create test result for unsupported performance test + test_result result(ggml_backend_name(backend), current_op_name, vars(), "perf", false, false, + "not supported"); + + output_printer->print_test_result(result); + + return true; + } + + ggml_backend_buffer_ptr buf_weights(nullptr); + if (ctx_weights) { + buf_weights.reset(ggml_backend_alloc_ctx_tensors(ctx_weights.get(), backend)); + if (buf_weights == NULL) { + printf("failed to allocate weight tensors\n"); + return false; + } + ggml_backend_buffer_set_usage(buf_weights.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + } + + // allocate + ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend)); // smart ptr + + if (buf == NULL) { + printf("failed to allocate tensors\n"); + return false; + } + + // randomize tensors + initialize_tensors(ctx.get()); + if (ctx_weights) { + initialize_tensors(ctx_weights.get()); + } + + // build graph + ggml_cgraph * gf = ggml_new_graph_custom(ctx.get(), graph_nodes, false); + ggml_build_forward_expand(gf, out); + + // warmup run + ggml_status status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } + + // determine number of runs + int n_runs; + bool is_cpu = ggml_backend_dev_type(ggml_backend_get_device(backend)) == GGML_BACKEND_DEVICE_TYPE_CPU; + if (op_flops(out) > 0) { + // based on flops + const uint64_t GFLOP = 1000 * 1000 * 1000; + const uint64_t target_flops_cpu = 8ULL * GFLOP; + const uint64_t target_flops_gpu = 100ULL * GFLOP; + uint64_t target_flops = is_cpu ? target_flops_cpu : target_flops_gpu; + n_runs = (int)std::min(ggml_graph_size(gf) - ggml_graph_n_nodes(gf), target_flops / op_flops(out)) + 1; + } else { + // based on memory size + const size_t GB = 1ULL << 30; + const size_t target_size_cpu = 8 * GB; + const size_t target_size_gpu = 32 * GB; + size_t target_size = is_cpu ? target_size_cpu : target_size_gpu; + n_runs = (int)std::min(ggml_graph_size(gf) - ggml_graph_n_nodes(gf), target_size / op_size(out)) + 1; + } + + // duplicate the op + for (int i = 1; i < n_runs; i++) { + ggml_graph_add_node(gf, out); + } + + // calculate memory + size_t mem = n_runs * op_size(out); + auto tensor_op_size = [](ggml_tensor * t) { + size_t size = ggml_nbytes(t); + // add source tensors + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (t->src[i] != NULL) { + size += ggml_nbytes(t->src[i]); + } + } + return size; + }; + for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) { + if (ggml_is_view_op(ggml_graph_node(gf, i)->op) || ggml_graph_node(gf, i) == out) { + continue; + } + mem += tensor_op_size(ggml_graph_node(gf, i)); + } + + // run + int64_t total_time_us = 0; + int64_t total_mem = 0; + int total_runs = 0; + do { + int64_t start_time = ggml_time_us(); + ggml_status status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } + int64_t end_time = ggml_time_us(); + + total_time_us += end_time - start_time; + total_mem += mem; + total_runs += n_runs; + } while (total_time_us < 1000*1000); // run for at least 1 second + + // Create test result + double avg_time_us = (double) total_time_us / total_runs; + double calculated_flops = (op_flops(out) > 0) ? (op_flops(out) * total_runs) / (total_time_us / 1e6) : 0.0; + double calculated_bandwidth = + (op_flops(out) == 0) ? total_mem / (total_time_us / 1e6) / 1024.0 / 1024.0 / 1024.0 : 0.0; + size_t calculated_memory_kb = op_size(out) / 1024; + + test_result result(ggml_backend_name(backend), current_op_name, vars(), "perf", true, true, "", avg_time_us, + calculated_flops, calculated_bandwidth, calculated_memory_kb, total_runs); + + if (output_printer) { + output_printer->print_test_result(result); + } + + return true; + } + + bool eval_support(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) { + mode = MODE_SUPPORT; + + static const size_t graph_nodes = 8192; + + ggml_init_params params = { + /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead_custom(graph_nodes, false), + /* .mem_base = */ NULL, + /* .no_alloc = */ true, + }; + ggml_context_ptr ctx(ggml_init(params)); // smart ptr + GGML_ASSERT(ctx); + + gf = ggml_new_graph_custom(ctx.get(), graph_nodes, false); + + ggml_tensor * out = build_graph(ctx.get()); + current_op_name = op_desc(out); + + if (!matches_filter(out, op_names_filter)) { + return true; + } + + bool supported = ggml_backend_supports_op(backend, out); + + std::string device_desc = ggml_backend_dev_description(ggml_backend_get_device(backend)); + std::string backend_reg_name = ggml_backend_reg_name(ggml_backend_dev_backend_reg(ggml_backend_get_device(backend))); + + test_result result(ggml_backend_name(backend), current_op_name, vars(), "support", supported, supported, + supported ? "yes" : "no", 0.0, 0.0, 0.0, 0, 0, device_desc, backend_reg_name); + + output_printer->print_test_result(result); + + return true; + } + + bool eval_grad(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) { + mode = MODE_GRAD; + const std::vector expect = grad_expect(); + + ggml_init_params params = { + /* .mem_size = */ ggml_tensor_overhead()*128 + 2*ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE, true), + /* .mem_base = */ NULL, + /* .no_alloc = */ true, + }; + ggml_context_ptr ctx(ggml_init(params)); // smart ptr + GGML_ASSERT(ctx); + + gf = ggml_new_graph_custom(ctx.get(), GGML_DEFAULT_GRAPH_SIZE, true); + gb = ggml_new_graph_custom(ctx.get(), GGML_DEFAULT_GRAPH_SIZE, true); + + ggml_tensor * out = build_graph(ctx.get()); + + if (!matches_filter(out, op_names_filter) || out->op == GGML_OP_OPT_STEP_ADAMW) { + return true; + } + + if (out->type != GGML_TYPE_F32) { + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), + test_status_t::NOT_SUPPORTED, + out->name + std::string("->type != FP32"))); + return true; + } + + // Print operation info first + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend))); + + // check if the backend supports the ops + bool supported = true; + bool any_params = false; + std::string failure_reason; + + for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { + if (!ggml_backend_supports_op(backend, t)) { + supported = false; + failure_reason = ggml_backend_name(backend); + break; + } + if ((t->flags & GGML_TENSOR_FLAG_PARAM)) { + any_params = true; + if (t->type != GGML_TYPE_F32) { + supported = false; + failure_reason = std::string(t->name) + "->type != FP32"; + break; + } + } + } + if (!any_params) { + supported = false; + failure_reason = op_desc(out); + } + + if (!supported) { + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), + test_status_t::NOT_SUPPORTED, failure_reason)); + return true; + } + + int64_t ngrads = 0; + for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { + if (t->flags & GGML_TENSOR_FLAG_PARAM) { + ngrads += ggml_nelements(t); + } + } + if (ngrads > grad_nmax()) { + test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); + info.set_large_tensor_skip(); + output_printer->print_operation(info); + return true; + } + + + if (!ggml_is_scalar(out)) { + out = ggml_sum(ctx.get(), out); + ggml_set_name(out, "sum_of_out"); + } + ggml_set_loss(out); + + ggml_build_forward_expand(gf, out); + ggml_graph_cpy(gf, gb); + ggml_build_backward_expand(ctx.get(), gb, nullptr); + if (expect.size() != 1 || expect[0] != 0.0f) { + GGML_ASSERT(ggml_graph_n_nodes(gb) > ggml_graph_n_nodes(gf)); + for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { + GGML_ASSERT(!(t->flags & GGML_TENSOR_FLAG_PARAM) || ggml_graph_get_grad(gb, t)->op != GGML_OP_NONE); + } + } + + for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { + if (!ggml_backend_supports_op(backend, t)) { + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), + test_status_t::NOT_SUPPORTED, + ggml_backend_name(backend))); + supported = false; + break; + } + if ((t->flags & GGML_TENSOR_FLAG_PARAM) && t->type != GGML_TYPE_F32) { + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), + test_status_t::NOT_SUPPORTED, + std::string(t->name) + "->type != FP32")); + supported = false; + break; + } + } + if (!supported) { + return true; + } + + // allocate + ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend)); // smart ptr + if (buf == NULL) { + test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); + info.set_error("allocation", ""); + output_printer->print_operation(info); + return false; + } + + initialize_tensors(ctx.get()); // Randomizes all tensors (including gradients). + ggml_graph_reset(gb); // Sets gradients to 1 if loss, 0 otherwise. + + ggml_status status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } + status = ggml_backend_graph_compute(backend, gb); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } + + bool ok = true; + for (struct ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) { + if (!(t->flags & GGML_TENSOR_FLAG_PARAM)) { + continue; + } + + const char * bn = ggml_backend_name(backend); + const int64_t ne = ggml_nelements(t); + + std::vector ga; + struct ggml_tensor * grad = ggml_graph_get_grad(gb, t); + if (grad) { + ga = tensor_to_float(grad); + } else { + ga.resize(ne); // default value is 0.0f + } + + for (int64_t i = 0; i < ne; ++i) { // gradient algebraic + // check for nans + if (!std::isfinite(ga[i])) { + test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); + info.set_gradient_info(i, bn, ga[i]); + output_printer->print_operation(info); + ok = false; + break; + } + } + if (!ok) { + break; + } + + std::vector gn(ne); // gradient numeric + GGML_ASSERT(ga.size() == gn.size()); + + std::vector x0 = tensor_to_float(t); // original t data + GGML_ASSERT(ggml_is_scalar(out)); + GGML_ASSERT(out->type == GGML_TYPE_F32); + + const float eps = grad_eps(); + for (int64_t i = 0; i < ne; ++i) { + const float xiu = x0[i] + 1.0f*eps; // x, index i, up + const float xiuh = x0[i] + 0.5f*eps; // x, index i, up half + const float xidh = x0[i] - 0.5f*eps; // x, index i, down half + const float xid = x0[i] - 1.0f*eps; // x, index i, down + + float fu, fuh, fdh, fd; // output values for xiu, xiuh, xid, xidh + + ggml_backend_tensor_set(t, &xiu, i*sizeof(float), sizeof(float)); + status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } + ggml_backend_tensor_get(out, &fu, 0, ggml_nbytes(out)); + + ggml_backend_tensor_set(t, &xid, i*sizeof(float), sizeof(float)); + status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } + ggml_backend_tensor_get(out, &fd, 0, ggml_nbytes(out)); + + if (grad_precise()) { + ggml_backend_tensor_set(t, &xiuh, i*sizeof(float), sizeof(float)); + status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } + ggml_backend_tensor_get(out, &fuh, 0, ggml_nbytes(out)); + + ggml_backend_tensor_set(t, &xidh, i*sizeof(float), sizeof(float)); + status = ggml_backend_graph_compute(backend, gf); + if (status != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + return false; + } + ggml_backend_tensor_get(out, &fdh, 0, ggml_nbytes(out)); + + gn[i] = (8.0*(double)fuh + (double)fd - (8.0*(double)fdh + (double)fu)) / (6.0*(double)eps); + } else { + gn[i] = (fu - fd) / (2.0f*eps); + } + + ggml_backend_tensor_set(t, x0.data(), 0, ggml_nbytes(t)); + } + + const double err = mean_abs_asymm(gn.data(), ga.data(), gn.size(), expect); + if (err > max_maa_err()) { + test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); + info.set_maa_error(err, max_maa_err()); + output_printer->print_operation(info); + ok = false; + break; + } + if (!ok) { + break; + } + } + + // Create final test result + test_operation_info final_info(op_desc(out), vars(), ggml_backend_name(backend)); + if (!ok) { + final_info.set_compare_failure(); + } + final_info.status = ok ? test_status_t::OK : test_status_t::FAIL; + output_printer->print_operation(final_info); + + if (ok) { + return true; + } + + return false; + } +}; + + +// #################################### +// ## Section 2: GGML Op Definitions ## +// #################################### + + +// The following is an example showing the bare minimum for creating a test for a GGML op. + +// GGML_OP_EXAMPLE +struct test_example : public test_case { + // Always define these 2 or variants thereof: + const ggml_type type; // The type of the input tensors. + const std::array ne; // The shape of the input tensors. + // For some ops it's necessary to define multiple types or shapes for the inputs. + // Or they may need additional parameters. + + // Put all parameters needed to fully define the test into one of the VARS_TO_STR macros. + // In most cases these are just the properties of the struct that you defined above. + // This is needed for info prints. + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + // Define a constructor for the struct. + // In most cases it will be sufficient to have the same arguments as the struct has properties + // and just use initializer lists. + test_example(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 5, 4, 3}) + : type(type), ne(ne) {} + + // Define how a simple GGML compute graph can be constructed for the new GGML op. + ggml_tensor * build_graph(ggml_context * ctx) override { + // Step 1: create input tensors that don't depend on any other tensors: + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); // Setting names is optional but it's useful for debugging. + + ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(b, "b"); + + // Step 2: use the op that you want to test in the GGML compute graph. + ggml_tensor * out = ggml_add(ctx, a, b); // For this example we're just doing a simple addition. + ggml_set_name(out, "out"); + + // Step 3: return the output tensor. + return out; + } + // In order to also check the gradients for your op, add calls like ggml_set_param(a) + // immediately after you create the tensors. + // This is optional and only makes sense if a backward pass has actually been implemented for the new op. +}; + + +// GGML_OP_UNARY +struct test_unary : public test_case { + const ggml_unary_op op; + const ggml_type type; + const std::array ne_a; + int v; // view (1 : non-contiguous a) + + std::string vars() override { + return VARS_TO_STR3(type, ne_a, v); + } + + test_unary(ggml_unary_op op, + ggml_type type = GGML_TYPE_F32, + std::array ne_a = {128, 2, 2, 2}, + int v = 0) + : op(op), type(type), ne_a(ne_a), v(v) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const bool grad_supported = op == GGML_UNARY_OP_ABS || op == GGML_UNARY_OP_SGN || op == GGML_UNARY_OP_NEG || + op == GGML_UNARY_OP_STEP || op == GGML_UNARY_OP_RELU || op == GGML_UNARY_OP_SILU || + op == GGML_UNARY_OP_EXPM1 || op == GGML_UNARY_OP_SOFTPLUS; + + ggml_tensor * a; + if (v & 1) { + auto ne = ne_a; + ne[0] *= 3; + ne[1] *= 2; + ne[2] *= 5; + ne[3] *= 4; + a = ggml_new_tensor(ctx, type, 4, ne.data()); + if (grad_supported) { + ggml_set_param(a); + } + ggml_set_name(a, "a"); + + a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(a, "view_of_a"); + } else { + a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + if (grad_supported) { + ggml_set_param(a); + } + ggml_set_name(a, "a"); + } + + ggml_tensor * out = ggml_unary(ctx, a, op); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + float min = -150.f; + float max = 150.f; + + // Keep FP16 exp/expm1 inputs in-range so all backends stay finite instead of + // disagreeing on whether overflow saturates to max-F16 or produces +inf. + if (type == GGML_TYPE_F16 && (op == GGML_UNARY_OP_EXP || op == GGML_UNARY_OP_EXPM1)) { + min = -10.f; + max = 10.f; + } + + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + // test extended range of values to check for NaNs in GELU + init_tensor_uniform(t, min, max); + } + } + + float grad_eps() override { + return 15.0f; + } + + std::vector grad_expect() override { + if (op == GGML_UNARY_OP_ABS) { + return {-1.0f, 1.0f}; + } + if (op == GGML_UNARY_OP_SGN || op == GGML_UNARY_OP_STEP) { + return {0.0f}; + } + if (op == GGML_UNARY_OP_RELU) { + return {0.0f, 1.0f}; + } + return {}; + } + +}; + +// GGML_OP_GLU +struct test_glu : public test_case { + const ggml_glu_op op; + const ggml_type type; + const std::array ne_a; + int v; // view (1 : non-contiguous a) + bool swapped; + + std::string vars() override { + return VARS_TO_STR4(type, ne_a, v, swapped); + } + + test_glu(ggml_glu_op op, + ggml_type type = GGML_TYPE_F32, + std::array ne_a = {128, 2, 2, 2}, + int v = 0, + bool swapped = false) + : op(op), type(type), ne_a(ne_a), v(v), swapped(swapped) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a; + if (v & 1) { + auto ne = ne_a; ne[0] *= 3; + a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(a, "view_of_a"); + } else { + a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_name(a, "a"); + } + + ggml_tensor * out = ggml_glu(ctx, a, op, swapped); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + // test extended range of values to check for NaNs in GELU + init_tensor_uniform(t, -150.f, 150.f); + } + } +}; + +struct test_glu_split : public test_case { + const ggml_glu_op op; + const ggml_type type; + const std::array ne_a; + int v; // view (1 : non-contiguous a) + + std::string vars() override { + return VARS_TO_STR3(type, ne_a, v) + ",split"; + } + + test_glu_split(ggml_glu_op op, + ggml_type type = GGML_TYPE_F32, + std::array ne_a = {128, 2, 2, 2}, + int v = 0) + : op(op), type(type), ne_a(ne_a), v(v) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a; + ggml_tensor * b; + if (v & 1) { + auto ne = ne_a; ne[0] *= 3; + a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(a, "view_of_a"); + + b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(b); + ggml_set_name(b, "b"); + + b = ggml_view_4d(ctx, b, ne_a[0], ne_a[1], ne_a[2], ne_a[3], b->nb[1], b->nb[2], b->nb[3], 0); + ggml_set_name(a, "view_of_b"); + } else { + a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + b = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_param(b); + ggml_set_name(b, "b"); + } + + ggml_tensor * out = ggml_glu_split(ctx, a, b, op); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + // test extended range of values to check for NaNs in GELU + init_tensor_uniform(t, -150.f, 150.f); + } + } +}; + +struct test_swiglu_oai : public test_case { + const ggml_type type; + const std::array ne_a; + int v; // view (1 : non-contiguous a) + float alpha; + float limit; + + std::string vars() override { + return VARS_TO_STR5(type, ne_a, v, alpha, limit); + } + + test_swiglu_oai(ggml_type type = GGML_TYPE_F32, + std::array ne_a = {128, 2, 2, 2}, + int v = 0, + float alpha = 1.702f, + float limit = 7.0f) + : type(type), ne_a(ne_a), v(v), alpha(alpha), limit(limit) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a; + ggml_tensor * b; + if (v & 1) { + auto ne = ne_a; ne[0] *= 3; + a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(a, "view_of_a"); + + b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(b); + ggml_set_name(b, "b"); + + b = ggml_view_4d(ctx, b, ne_a[0], ne_a[1], ne_a[2], ne_a[3], b->nb[1], b->nb[2], b->nb[3], 0); + ggml_set_name(a, "view_of_b"); + } else { + a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + b = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_param(b); + ggml_set_name(b, "b"); + } + + ggml_tensor * out = ggml_swiglu_oai(ctx, a, b, alpha, limit); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + // test extended range of values to check for NaNs in GELU + init_tensor_uniform(t, -150.f, 150.f); + } + } +}; + +// GGML_OP_GET_ROWS +struct test_get_rows : public test_case { + const ggml_type type; + const int n; // cols + const int m; // rows + const int r; // rows to get + const int be1; // batch size + const int be2; // batch size + const bool v; // view (non-contiguous src1) + + std::string vars() override { + return VARS_TO_STR7(type, n, m, r, be1, be2, v); + } + + test_get_rows(ggml_type type = GGML_TYPE_F32, int n = 10, int m = 5, int r = 3, int be1 = 1, int be2 = 1, bool v = false) + : type(type), n(n), m(m), r(r), be1(be1), be2(be2), v(v) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * in = ggml_new_tensor_4d(ctx, type, n, m, be1, be2); + ggml_set_name(in, "in"); + + ggml_tensor * rows = ggml_new_tensor_3d(ctx, GGML_TYPE_I32, r, be1, be2); + ggml_set_name(rows, "rows"); + if (v) { + rows = ggml_view_3d(ctx, rows, r/2, be1, be2, rows->nb[1], rows->nb[2], 0); + ggml_set_name(rows, "view_of_rows"); + } + + const bool grad_supported = ggml_is_matrix(in) && ggml_is_vector(rows); + if (grad_supported) { + ggml_set_param(in); + // rows is a constant input -> no gradients + } + + ggml_tensor * out = ggml_get_rows(ctx, in, rows); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_I32) { + if (ggml_is_view_op(t->op)) { continue; } + // rows + std::vector data(r*be1*be2); + for (int i = 0; i < r*be1*be2; i++) { + data[i] = rand() % m; + } + ggml_backend_tensor_set(t, data.data(), 0, r * be1 * be2 * sizeof(int)); + } else { + init_tensor_uniform(t); + } + } + } +}; + +// GGML_OP_GET_ROWS_BACK +struct test_get_rows_back : public test_case { + const ggml_type type; + const int n; // cols + const int m; // rows + const int r; // rows to get + const int b; // batch size + const bool v; // view (non-contiguous src1) + + std::string vars() override { + return VARS_TO_STR6(type, n, m, r, b, v); + } + + test_get_rows_back(ggml_type type = GGML_TYPE_F32, int n = 10, int m = 5, int r = 3, int b = 1, bool v = false) + : type(type), n(n), m(m), r(r), b(b), v(v) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * in_forward = ggml_new_tensor_3d(ctx, type, n, m, b); + ggml_set_name(in_forward, "in_forward"); + + ggml_tensor * rows = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, r, b); + ggml_set_name(rows, "rows"); + if (v) { + rows = ggml_view_2d(ctx, rows, r/2, b, rows->nb[1], 0); + ggml_set_name(rows, "view_of_rows"); + } + + ggml_tensor * grad = ggml_new_tensor_3d(ctx, type, n, r, b); + ggml_set_name(grad, "grad"); + + ggml_tensor * out = ggml_get_rows_back(ctx, grad, rows, in_forward); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_I32) { + if (ggml_is_view_op(t->op)) { continue; } + // rows + std::vector data(r*b); + for (int i = 0; i < r*b; i++) { + data[i] = rand() % m; + } + ggml_backend_tensor_set(t, data.data(), 0, r * b * sizeof(int)); + } else { + init_tensor_uniform(t); + } + } + } +}; + +static void init_set_rows_row_ids(ggml_tensor * t, int num_rows) { + std::random_device rd; + std::default_random_engine rng(rd()); + for (int i2 = 0; i2 < t->ne[2]; i2++) { + for (int i1 = 0; i1 < t->ne[1]; i1++) { + // generate a shuffled subset of row indices + std::vector data(num_rows); + for (int i = 0; i < num_rows; i++) { + data[i] = i; + } + std::shuffle(data.begin(), data.end(), rng); + data.resize(t->ne[0]); + + const size_t offs = i1*t->nb[1] + i2*t->nb[2]; + if (t->type == GGML_TYPE_I32) { + // TODO: Make a template or something + std::vector data_i32(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + data_i32[i] = static_cast(data[i]); + } + ggml_backend_tensor_set(t, data_i32.data(), offs, t->ne[0]*sizeof(int32_t)); + } else { + ggml_backend_tensor_set(t, data.data(), offs, t->ne[0]*sizeof(int64_t)); + } + } + } +} + +// GGML_OP_SET_ROWS +struct test_set_rows : public test_case { + const ggml_type type_src; + const ggml_type type_dst; + const ggml_type type_idx; + const std::array ne; + const std::array nr23; // broadcast only dims 2 and 3 + const int r; // rows to set + const bool v; // view (non-contiguous src1) + + std::string vars() override { + return VARS_TO_STR7(type_src, type_dst, type_idx, ne, nr23, r, v); + } + + test_set_rows(ggml_type type_src, + ggml_type type_dst, + ggml_type type_idx, + std::array ne, + std::array nr23, + int r, bool v = false) + : type_src(type_src), type_dst(type_dst), type_idx(type_idx), ne(ne), nr23(nr23), r(r), v(v) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * dst = ggml_new_tensor_4d(ctx, type_dst, ne[0], ne[1], ne[2]*nr23[0], ne[3]*nr23[1]); + ggml_set_name(dst, "dst"); + + ggml_tensor * src = ggml_new_tensor_4d(ctx, type_src, ne[0], r, ne[2]*nr23[0], ne[3]*nr23[1]); + ggml_set_name(src, "src"); + + ggml_tensor * row_idxs = ggml_new_tensor_3d(ctx, type_idx, r, ne[2], ne[3]); + ggml_set_name(row_idxs, "row_idxs"); + + if (v) { + src = ggml_view_4d(ctx, src, ne[0], r/2, ne[2]*nr23[0], ne[3]*nr23[1], src->nb[1], src->nb[2], src->nb[3], 0); + row_idxs = ggml_view_3d(ctx, row_idxs, r/2, ne[2], ne[3], row_idxs->nb[1], row_idxs->nb[2], 0); + ggml_set_name(row_idxs, "view_of_rows"); + } + + ggml_tensor * out = ggml_set_rows(ctx, dst, src, row_idxs); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_I64 || t->type == GGML_TYPE_I32) { + if (ggml_is_view_op(t->op)) { + continue; + } + + init_set_rows_row_ids(t, ne[1]); + } else { + init_tensor_uniform(t); + } + } + } + + double max_nmse_err() override { + if (type_dst == GGML_TYPE_Q4_0 || type_dst == GGML_TYPE_Q4_1 || type_dst == GGML_TYPE_IQ4_NL || + type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1 || type_dst == GGML_TYPE_Q8_0) { + // estimate what the max nmse error would be if one quantized value is + // off by one. The test values are distributed in [-1,1], so it'll be + // roughly (2.0 / 2^bits)^2, divided by the mean square value of the reference, + // which is roughly 0.25 times the number of elements. + double err_estimate = 1.0f/8.0f; + if (type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1) { + err_estimate /= 2.0f; + } + if (type_dst == GGML_TYPE_Q8_0) { + err_estimate /= 8.0f; + } + err_estimate *= err_estimate; + err_estimate /= 0.25f*float(ne[0] * r * ne[2]*nr23[0] * ne[3]*nr23[1]); + return err_estimate; + } + return 1e-7; + } + + // See dicussion here: https://github.com/ggml-org/llama.cpp/pull/23760#issuecomment-4566312209 + double max_nmse_err(ggml_backend_t backend) override { + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); + if (type_dst == GGML_TYPE_Q8_0 && strcmp(ggml_backend_reg_name(reg), "WebGPU") == 0) { + return std::max(test_case::max_nmse_err(backend), 2e-7); + } + return test_case::max_nmse_err(backend); + } +}; + +// GGML_OP_ROPE + GGML_OP_VIEW + GGML_OP_SET_ROWS +struct test_rope_set_rows : public test_case { + const ggml_type type; + const ggml_type type_idx; + const std::array ne_a; + int mode; + const int n_ctx{512}; + const int n_dims{128}; + + std::string vars() override { + return VARS_TO_STR4(type, type_idx, ne_a, mode); + } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "ROPE_SET_ROWS"; + } + + bool run_whole_graph() override { return true; } + + test_rope_set_rows(ggml_type type, + ggml_type type_idx, + std::array ne_a, + int mode) + : type(type), type_idx(type_idx), ne_a(ne_a), mode(mode) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne_a[0], ne_a[1], ne_a[2], 1); + ggml_set_name(a, "a"); + + const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; + const bool is_vision = mode == GGML_ROPE_TYPE_VISION; + + ggml_tensor * pos; + if (is_mrope || is_vision) { + pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne_a[2] * 4); + } else { + pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne_a[2]); + } + ggml_set_name(pos, "pos"); + + float fs = 1.4245f; + float ef = 0.7465f; + float af = 1.4245f; + ggml_tensor * freq = nullptr; + + ggml_tensor * rope = nullptr; + if (is_mrope) { + if (is_vision) { + GGML_ASSERT(n_dims/4 > 0); + int rope_sections[4] = {n_dims/4, n_dims/4, 0, 0}; // Vision-RoPE only use first two dimension for image (x, y) coordinate + rope = ggml_rope_multi(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + } else { + GGML_ASSERT(n_dims/3 > 0); + int rope_sections[4] = {n_dims/3, n_dims/3, n_dims/3, 0}; + rope = ggml_rope_multi(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + } + } else { + rope = ggml_rope(ctx, a, pos, ne_a[0], mode); + } + + ggml_tensor * view = ggml_view_2d(ctx, rope, ne_a[0] * ne_a[1], ne_a[2], rope->nb[2], 0); + + ggml_tensor * dst = ggml_new_tensor_4d(ctx, type, ne_a[0] * ne_a[1], ne_a[2] * ne_a[3], 1, 1); + ggml_set_name(dst, "dst"); + + ggml_tensor * row_idxs = ggml_new_tensor_3d(ctx, type_idx, ne_a[2], 1, 1); + ggml_set_name(row_idxs, "row_idxs"); + + ggml_tensor * out = ggml_set_rows(ctx, dst, view, row_idxs); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "row_idxs") == 0) { + if (ggml_is_view_op(t->op)) { + continue; + } + init_set_rows_row_ids(t, ne_a[2]); + } else if (t->type == GGML_TYPE_I32) { + // pos + const int num_pos_ids = (mode & GGML_ROPE_TYPE_MROPE) ? ne_a[2] * 4 : ne_a[2]; + std::vector data(num_pos_ids); + for (int i = 0; i < num_pos_ids; i++) { + data[i] = rand() % n_ctx; + } + ggml_backend_tensor_set(t, data.data(), 0, num_pos_ids * sizeof(int)); + } else { + if (t->ne[0] == n_dims/2) { + // frequency factors in the range [0.9f, 1.1f] + init_tensor_uniform(t, 0.9f, 1.1f); + } else { + init_tensor_uniform(t); + } + } + } + } +}; + +// GGML_OP_RMS_NORM + GGML_OP_MUL + GGML_OP_ROPE (+ GGML_OP_VIEW + GGML_OP_SET_ROWS) +struct test_rms_norm_mul_rope : public test_case { + const std::array ne; + const float eps; + const bool multi_add; // test a sequence of adds feeding into rms_norm + const bool set_rows; + int mode; + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "RMS_NORM_MUL_ROPE"; + } + + bool run_whole_graph() override { return true; } + + std::string vars() override { + return VARS_TO_STR5(ne, eps, multi_add, set_rows, mode); + } + + test_rms_norm_mul_rope(std::array ne, float eps = 1e-6f, bool multi_add = false, + bool set_rows = false, int mode = GGML_ROPE_TYPE_NORMAL) + : ne(ne), eps(eps), multi_add(multi_add), set_rows(set_rows), mode(mode) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], 1); + ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], 1); + ggml_tensor * c = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], 1); + + if (multi_add) { + a = ggml_add(ctx, ggml_add(ctx, a, b), c); + } + + a = ggml_mul(ctx, ggml_rms_norm(ctx, a, eps), b); + + ggml_tensor * pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne[2]); + + ggml_tensor * rope = ggml_rope(ctx, a, pos, ne[0], mode); + + ggml_tensor * out; + + if (set_rows) { + ggml_tensor * view = ggml_view_2d(ctx, rope, ne[0] * ne[1], ne[2], rope->nb[2], 0); + + ggml_tensor * dst = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, ne[0] * ne[1], ne[2] * ne[3], 1, 1); + ggml_set_name(dst, "dst"); + + ggml_tensor * row_idxs = ggml_new_tensor_3d(ctx, GGML_TYPE_I64, ne[2], 1, 1); + ggml_set_name(row_idxs, "row_idxs"); + + out = ggml_set_rows(ctx, dst, view, row_idxs); + ggml_set_name(out, "out"); + } else { + out = rope; + } + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_I64 || t->type == GGML_TYPE_I32) { + if (ggml_is_view_op(t->op)) { + continue; + } + + init_set_rows_row_ids(t, ne[2]); + } else { + init_tensor_uniform(t); + } + } + } +}; + +// GGML_OP_ARGMAX +struct test_argmax : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_argmax(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 100, 1, 1}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_argmax(ctx, a); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + std::random_device rd; + std::default_random_engine rng(rd()); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_F32) { + // initialize with unique values to avoid ties + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector data(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + data[i] = i; + } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(float)); + } + } else { + init_tensor_uniform(t); + } + } + } + + double max_nmse_err() override { + return 0.0; + } +}; + +// GGML_OP_COUNT_EQUAL +struct test_count_equal : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_count_equal(ggml_type type = GGML_TYPE_F32, + std::array ne = {4, 500, 1, 1}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * a_argmax = ggml_argmax(ctx, a); + ggml_set_name(a_argmax, "a_argmax"); + + ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(b, "b"); + + ggml_tensor * b_argmax = ggml_argmax(ctx, b); + ggml_set_name(b_argmax, "b_argmax"); + + ggml_tensor * out = ggml_count_equal(ctx, a_argmax, b_argmax); + ggml_set_name(out, "out"); + + return out; + } + + double max_nmse_err() override { + return 0.0; + } + + void initialize_tensors(ggml_context * ctx) override { + std::random_device rd; + std::default_random_engine rng(rd()); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_F32) { + // initialize with unique values to avoid ties + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector data(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + data[i] = i; + } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(float)); + } + } else { + init_tensor_uniform(t); + } + } + } +}; + +// GGML_OP_REPEAT +struct test_repeat : public test_case { + const ggml_type type; + const std::array ne; + const std::array nr; + + std::string vars() override { + return VARS_TO_STR3(type, ne, nr); + } + + size_t op_size(ggml_tensor * t) override { + return ggml_nbytes(t) * 2; + } + + test_repeat(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 5, 4, 3}, + std::array nr = {2, 2, 2, 2}) + : type(type), ne(ne), nr(nr) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * target = ggml_new_tensor_4d(ctx, type, ne[0]*nr[0], ne[1]*nr[1], ne[2]*nr[2], ne[3]*nr[3]); + ggml_set_name(target, "target"); + + ggml_tensor * src = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(src); + ggml_set_name(src, "src"); + + ggml_tensor * out = ggml_repeat(ctx, src, target); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_REPEAT_BACK +struct test_repeat_back : public test_case { + const ggml_type type; + const std::array ne; + const std::array nr; + const bool v; // whether src is a noncontiguous view + + std::string vars() override { + return VARS_TO_STR4(type, ne, nr, v); + } + + size_t op_size(ggml_tensor * t) override { + return ggml_nbytes(t) * 2; + } + + test_repeat_back(ggml_type type = GGML_TYPE_F32, + std::array ne = {8, 6, 4, 2}, + std::array nr = {2, 2, 2, 2}, + bool v = false) + : type(type), ne(ne), nr(nr), v(v) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * src = ggml_new_tensor_4d(ctx, type, ne[0]*nr[0], ne[1]*nr[1], ne[2]*nr[2], ne[3]*nr[3]); + ggml_set_name(src, "src"); + + if (v) { + GGML_ASSERT(ne[0] % 2 == 0); + GGML_ASSERT(ne[1] % 2 == 0); + GGML_ASSERT(ne[2] % 2 == 0); + GGML_ASSERT(ne[3] % 2 == 0); + GGML_ASSERT(nr[0] % 2 == 0 || nr[0] == 1); + GGML_ASSERT(nr[1] % 2 == 0 || nr[1] == 1); + GGML_ASSERT(nr[2] % 2 == 0 || nr[2] == 1); + GGML_ASSERT(nr[3] % 2 == 0 || nr[3] == 1); + + const int64_t ne00 = nr[0] == 1 ? src->ne[0] : src->ne[0] / 2; + const int64_t ne01 = nr[1] == 1 ? src->ne[1] : src->ne[1] / 2; + const int64_t ne02 = nr[2] == 1 ? src->ne[2] : src->ne[2] / 2; + const int64_t ne03 = nr[3] == 1 ? src->ne[3] : src->ne[3] / 2; + + src = ggml_view_4d(ctx, src, ne00, ne01, ne02, ne03, src->nb[1], src->nb[2], src->nb[3], 0); + } + + ggml_tensor * target = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(target, "target"); + + ggml_tensor * out = ggml_repeat_back(ctx, src, target); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_DUP +struct test_dup : public test_case { + const ggml_type type; + const std::array ne; + const std::array permute; + bool _use_permute; + + std::string vars() override { + std::string v = VARS_TO_STR2(type, ne); + if (_use_permute) v += "," + VAR_TO_STR(permute); + return v; + } + + test_dup(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 10, 20, 1}, + std::array permute = {0, 0, 0, 0}) + : type(type), ne(ne), permute(permute), + _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * src = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(src); + ggml_set_name(src, "src"); + + if (_use_permute) { + src = ggml_permute(ctx, src, permute[0], permute[1], permute[2], permute[3]); + ggml_set_name(src, "src_permuted"); + } + + ggml_tensor * out = ggml_dup(ctx, src); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_SET +struct test_set : public test_case { + const ggml_type type_src; + const ggml_type type_dst; + const std::array ne; + const int dim; + const bool inplace; + + std::string vars() override { + return VARS_TO_STR5(type_src, type_dst, ne, dim, inplace); + } + + size_t op_size(ggml_tensor * t) override { + return ggml_nbytes(t) + ggml_nbytes(t->src[0]); + } + + test_set(ggml_type type_src = GGML_TYPE_F32, ggml_type type_dst = GGML_TYPE_F32, + std::array ne = {6, 5, 4, 3}, int dim = 1, bool inplace = false) + : type_src(type_src), type_dst(type_dst), ne(ne), dim(dim), inplace(inplace) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * src = ggml_new_tensor(ctx, type_src, 4, ne.data()); + ggml_set_param(src); + ggml_set_name(src, "src"); + + auto ne_dst = ne; + for (int i = 0; i < dim; ++i) { + ne_dst[i] *= 2; + } + ggml_tensor * dst = ggml_new_tensor(ctx, type_dst, 4, ne_dst.data()); + ggml_set_param(dst); + ggml_set_name(dst, "dst"); + + size_t offset = 0; + for (int i = 0; i < dim; ++i) { + offset += ((ne_dst[i] - ne[i])/2)*dst->nb[i]; + } + ggml_tensor * out; + if (inplace) { + out = ggml_set_inplace(ctx, dst, src, + // The backward pass requires setting a contiguous region: + src->nb[1], src->nb[2], src->nb[3], offset); + } else { + out = ggml_set(ctx, dst, src, + // The backward pass requires setting a contiguous region: + src->nb[1], src->nb[2], src->nb[3], offset); + } + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_CPY +struct test_cpy : public test_case { + const ggml_type type_src; + const ggml_type type_dst; + const std::array ne_src; + const std::array ne_dst; + const std::array permute_src; + const std::array permute_dst; + const std::array dst_alloc; // if set, dst is a view into a larger buffer (strided) + bool _src_use_permute; + bool _dst_use_permute; + bool _src_transpose; + bool _use_dst_shape; + bool _use_dst_alloc; + + std::string vars() override { + if (_use_dst_alloc) { + return VARS_TO_STR8(type_src, type_dst, ne_src, ne_dst, permute_src, permute_dst, _src_transpose, dst_alloc); + } + if (_use_dst_shape) { + return VARS_TO_STR7(type_src, type_dst, ne_src, ne_dst, permute_src, permute_dst, _src_transpose); + } + return VARS_TO_STR6(type_src, type_dst, ne_src, permute_src, permute_dst, _src_transpose); + } + + int64_t total_elements() const { + return ne_src[0] * ne_src[1] * ne_src[2] * ne_src[3]; + } + + double max_nmse_err() override { + if (type_src == type_dst) { + return 0.0; + } + if (type_dst == GGML_TYPE_Q4_0 || type_dst == GGML_TYPE_Q4_1 || type_dst == GGML_TYPE_IQ4_NL || + type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1 || type_dst == GGML_TYPE_Q8_0) { + // estimate what the max nmse error would be if one quantized value is + // off by one. The test values are distributed in [-150,150], so it'll be + // roughly (150*2.0 / 2^bits)^2, divided by the mean square value of the reference, + // which is roughly 0.25*150^2 times the number of elements. + double err_estimate = 1.0f/8.0f * 150.0f; + if (type_dst == GGML_TYPE_IQ4_NL) { + // iq4_nl values are a bit more spread out + err_estimate *= 2.0f; + } + if (type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1) { + err_estimate /= 2.0f; + } + if (type_dst == GGML_TYPE_Q8_0) { + err_estimate /= 8.0f; + } + err_estimate *= err_estimate; + err_estimate /= (150.0f*150.0f*0.25f)*float(total_elements()); + return err_estimate; + } + return 1e-6; + } + + size_t op_size(ggml_tensor * t) override { + return ggml_nbytes(t) + ggml_nbytes(t->src[0]); + } + + test_cpy(ggml_type type_src = GGML_TYPE_F32, ggml_type type_dst = GGML_TYPE_F32, + std::array ne_src = {10, 10, 10, 1}, + std::array ne_dst = {-1, -1, -1, -1}, + std::array permute_src = {0, 0, 0, 0}, + std::array permute_dst = {0, 0, 0, 0}, + bool transpose_src = false, + std::array dst_alloc = {0, 0, 0, 0}) + : type_src(type_src), type_dst(type_dst), ne_src(ne_src), ne_dst(ne_dst), permute_src(permute_src), permute_dst(permute_dst), + dst_alloc(dst_alloc), + _src_use_permute(permute_src[0] + permute_src[1] + permute_src[2] + permute_src[3] > 0), + _dst_use_permute(permute_dst[0] + permute_dst[1] + permute_dst[2] + permute_dst[3] > 0), + _src_transpose(transpose_src), + _use_dst_shape(ne_dst[0] >= 0 && ne_dst[1] >= 0 && ne_dst[2] >= 0 && ne_dst[3] >= 0), + _use_dst_alloc(dst_alloc[0] > 0){} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * src = ggml_new_tensor(ctx, type_src, 4, ne_src.data()); + ggml_set_param(src); + ggml_set_name(src, "src"); + + if (_src_use_permute) { + src = ggml_permute(ctx, src, permute_src[0], permute_src[1], permute_src[2], permute_src[3]); + ggml_set_name(src, "src_permuted"); + } + + if (_src_transpose) { + src = ggml_transpose(ctx, src); + ggml_set_name(src, "src_transposed"); + } + + std::array dst_ne = _use_dst_shape ? ne_dst : std::array{src->ne[0], src->ne[1], src->ne[2], src->ne[3]}; + ggml_tensor * dst; + + if (_use_dst_alloc) { + // view a sub-block of a larger buffer -> strided dst + ggml_tensor * dst_buf = ggml_new_tensor(ctx, type_dst, 4, dst_alloc.data()); + ggml_set_name(dst_buf, "dst_buf"); + dst = ggml_view_4d(ctx, dst_buf, dst_ne[0], dst_ne[1], dst_ne[2], dst_ne[3], + dst_buf->nb[1], dst_buf->nb[2], dst_buf->nb[3], 0); + ggml_set_name(dst, "dst_view"); + } else { + dst = ggml_new_tensor(ctx, type_dst, 4, dst_ne.data()); + ggml_set_name(dst, "dst"); + + if (_dst_use_permute) { + dst = ggml_permute(ctx, dst, permute_dst[0], permute_dst[1], permute_dst[2], permute_dst[3]); + ggml_set_name(dst, "dst_permuted"); + } + } + + ggml_tensor * out = ggml_cpy(ctx, src, dst); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + // test extended range of values to check if casting between f32 and i32 is consistent + init_tensor_uniform(t, -150.f, 150.f); + } + } +}; + +// GGML_OP_CONT +struct test_cont : public test_case { + const ggml_type type; + const std::array ne; + bool use_view_slice; + + std::string vars() override { + return VARS_TO_STR3(type, ne, use_view_slice); + } + + test_cont(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 10, 10, 1}, + bool use_view_slice = false) + : type(type), ne(ne), use_view_slice(use_view_slice) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * src = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(src); + ggml_set_name(src, "src"); + + + ggml_tensor * dst; + if (use_view_slice) { + dst = ggml_view_4d(ctx, src, src->ne[0], 1, src->ne[2], src->ne[3], + src->nb[1], src->nb[2], src->nb[3], src->nb[0] * (src->ne[1] - 1)); + ggml_set_name(dst, "src_view_slice"); + } else { + dst = ggml_transpose(ctx, src); + ggml_set_name(dst, "src_transposed"); + } + + ggml_tensor * out = ggml_cont(ctx, dst); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_ADD +// GGML_OP_SUB +// GGML_OP_MUL +// GGML_OP_DIV +struct test_bin_bcast : public test_case { + using op_t = ggml_tensor * (*) (ggml_context *, ggml_tensor *, ggml_tensor *); + op_t op; + const ggml_type type; + const std::array ne; + const std::array nr; + int nf; // number of fused ops, nf == 1 -> single op (no fusion) + bool perm1; // permute src1? + bool src_overlap; // src0 and src1 are overlapping views of the same buffer + + bool run_whole_graph() override { return nf > 1; } + + std::string vars() override { + return VARS_TO_STR6(type, ne, nr, nf, perm1, src_overlap); + } + + size_t op_size(ggml_tensor * t) override { + return ggml_nbytes(t) * 3; + } + + test_bin_bcast(op_t op, ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 10, 1, 1}, + std::array nr = {1, 2, 1, 1}, + int nf = 1, + bool perm1 = false, bool src_overlap = false) + : op(op), type(type), ne(ne), nr(nr), nf(nf), perm1(perm1), src_overlap(src_overlap) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + GGML_ASSERT(nf <= 16); + + ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0]*nr[0], ne[1]*nr[1], ne[2]*nr[2], ne[3]*nr[3]); + ggml_set_name(a, "a"); + + ggml_tensor * b[16]; + for (int i = 0; i < nf; ++i) { + if (perm1) { + const int p[4] = { 1, 2, 0, 3 }; // hardcoded for now + + b[i] = ggml_new_tensor_4d(ctx, type, ne[p[0]], ne[p[1]], ne[p[2]], ne[p[3]]); + b[i] = ggml_permute(ctx, b[i], p[0], p[1], p[2], p[3]); + } else if (src_overlap) { + b[i] = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], 2 * (ne[3] / 3), a->nb[1], a->nb[2], a->nb[3], (ne[3] / 3) * a->nb[3]); + } else { + b[i] = ggml_new_tensor(ctx, type, 4, ne.data()); + } + ggml_set_name(b[i], (std::string("b") + std::to_string(i)).c_str()); + } + + // The backward pass supports broadcasting only for GGML_ADD: + const bool grad_supported = op == ggml_add && ggml_are_same_shape(a, b[0]) && nf == 1 && !perm1; + if (grad_supported) { + ggml_set_param(a); + ggml_set_param(b[0]); + } + + ggml_tensor *out; + + if (src_overlap) { + out = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], 2 * (ne[3] / 3), a->nb[1], a->nb[2], a->nb[3], 0); + } else { + out = a; + } + + for (int i = 0; i < nf; ++i) { + out = op(ctx, out, b[i]); + } + + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (op == ggml_mul || op == ggml_div) { + // MUL and DIV have numerical issues around zero: + init_tensor_uniform(t, 0.9f, 1.1f); + } else { + init_tensor_uniform(t); + } + } + } + + float grad_eps() override { + return 0.1f * (op == ggml_mul ? ne[0]*ne[1]*ne[2]*ne[3] : 1); + } + + bool grad_precise() override { + return op == ggml_div; + } + + double max_maa_err() override { + return op == ggml_add ? 1e-4 : 1e-3; + } +}; + +// GGML_OP_ADD_ID +struct test_add_id : public test_case { + const ggml_type type_a; + const ggml_type type_b; + const int64_t n_embd; + const int64_t n_experts; + const int64_t n_experts_used; + const int64_t n_token; + + std::string vars() override { + return VARS_TO_STR6(type_a, type_b, n_embd, n_experts, n_experts_used, n_token); + } + + size_t op_size(ggml_tensor * t) override { + return ggml_nbytes(t) + ggml_nbytes(t->src[0]) + ggml_nbytes(t->src[2]); + } + + test_add_id(ggml_type type_a = GGML_TYPE_F32, + ggml_type type_b = GGML_TYPE_F32, + int64_t n_embd = 128, + int64_t n_experts = 16, + int64_t n_experts_used = 8, + int64_t n_token = 10) + : type_a(type_a), type_b(type_b), n_embd(n_embd), + n_experts(n_experts), n_experts_used(n_experts_used), n_token(n_token) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_3d(ctx, type_a, n_embd, n_experts_used, n_token); + ggml_tensor * b = ggml_new_tensor_2d(ctx, type_b, n_embd, n_experts); + ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_experts, n_token); + if (n_experts_used != n_experts) { + ids = ggml_view_2d(ctx, ids, n_experts_used, n_token, ids->nb[1], 0); + ggml_set_name(ids, "view_of_ids"); + } + + ggml_tensor * out = ggml_add_id(ctx, a, b, ids); + ggml_set_name(out, "out"); + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_I32) { + if (ggml_is_view_op(t->op)) { continue; } + std::random_device rd; + std::default_random_engine rng(rd()); + // ids + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector data(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + data[i] = i % n_experts; + } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); + } + } else { + init_tensor_uniform(t); + } + } + } +}; + +// GGML_OP_SCALE +struct test_scale : public test_case { + const ggml_type type; + const std::array ne; + float scale; + float bias; + bool inplace; + + std::string vars() override { + return VARS_TO_STR5(type, ne, scale, bias, inplace); + } + + test_scale(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 10, 10, 10}, + float scale = 2.0f, + float bias = 0.0f, + bool inplace = false) + : type(type), ne(ne), scale(scale), bias(bias), inplace(inplace) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out; + if (inplace) { + out = ggml_scale_bias_inplace(ctx, a, scale, bias); + } else { + out = ggml_scale_bias(ctx, a, scale, bias); + } + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_SCALE + GGML_UNARY_OP_TANH + GGML_OP_SCALE +struct test_softcap : public test_case { + const ggml_type type; + const std::array ne; + float softcap; + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "SOFTCAP"; + } + + bool run_whole_graph() override { return true; } + + std::string vars() override { + return VARS_TO_STR3(type, ne, softcap); + } + + test_softcap(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 10, 10, 10}, + float softcap = 30.0f) + : type(type), ne(ne), softcap(softcap) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_scale(ctx, ggml_tanh(ctx, ggml_scale(ctx, a, 1.0f / softcap)), softcap); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_SILU_BACK +struct test_silu_back : public test_case { + const ggml_type type; + const std::array ne; + float eps; + + std::string vars() override { + return VARS_TO_STR3(type, ne, eps); + } + + test_silu_back(ggml_type type = GGML_TYPE_F32, + std::array ne = {64, 5, 4, 3}, + float eps = 1e-6f) + : type(type), ne(ne), eps(eps) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * grad = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(grad, "grad"); + + ggml_tensor * out = ggml_silu_back(ctx, a, grad); + ggml_set_name(out, "out"); + + return out; + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_NORM +struct test_norm : public test_case { + const ggml_type type; + const std::array ne; + const bool v; // whether a is a non-contiguous view + const float eps; + const bool noncontig_rows; + + std::string vars() override { + return VARS_TO_STR5(type, ne, v, eps, noncontig_rows); + } + + test_norm(ggml_type type = GGML_TYPE_F32, + std::array ne = {64, 5, 4, 3}, + bool v = false, + float eps = 1e-6f, + bool noncontig_rows = false) + : type(type), ne(ne), v(v), eps(eps), noncontig_rows(noncontig_rows) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const std::array ne_a = noncontig_rows ? + std::array{ ne[1], ne[0], ne[2], ne[3] } : ne; + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_name(a, "a"); + + if (noncontig_rows) { + a = ggml_permute(ctx, a, 1, 0, 2, 3); + ggml_set_name(a, "permuted a"); + } + if (v) { + a = ggml_view_4d(ctx, a, a->ne[0]/2, a->ne[1]/2, a->ne[2]/2, a->ne[3]/2, a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(a, "view of a"); + } + + ggml_tensor * out = ggml_norm(ctx, a, eps); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_NORM + GGML_OP_MUL + GGML_OP_ADD +struct test_norm_mul_add : public test_case { + const ggml_type type; + const std::array ne; + float eps; + const bool broadcast; + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "NORM_MUL_ADD"; + } + + bool run_whole_graph() override { return true; } + + std::string vars() override { + return VARS_TO_STR4(type, ne, eps, broadcast); + } + + test_norm_mul_add(ggml_type type = GGML_TYPE_F32, + std::array ne = {128, 2, 1, 1}, + float eps = 1e-5f, + bool broadcast = false) + : type(type), ne(ne), eps(eps), broadcast(broadcast) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + std::array broadcast_dims = {ne[0], ne[1] * 2, ne[2] * 2, ne[3] * 2}; + + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, broadcast ? broadcast_dims.data() : ne.data()); + ggml_tensor * w = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); ggml_set_param(w); ggml_set_param(b); + ggml_set_name(a, "a"); ggml_set_name(w, "w"); ggml_set_name(b, "b"); + + // Use a, w and b early to avoid OP_NONE in graph + a = ggml_add(ctx, ggml_add(ctx, a, w), b); + + ggml_tensor * n = ggml_norm(ctx, a, eps); + ggml_tensor * m = ggml_mul(ctx, n, w); + ggml_tensor * out = ggml_add(ctx, m, b); + ggml_set_name(out, "out"); + return out; + } +}; +// GGML_OP_RMS_NORM +struct test_rms_norm : public test_case { + const ggml_type type; + const std::array ne; + const bool v; // whether a is a non-contiguous view + const float eps; + const bool inplace; // whether to do the operation inplace + + std::string vars() override { + return VARS_TO_STR5(type, ne, v, eps, inplace); + } + + test_rms_norm(ggml_type type = GGML_TYPE_F32, + std::array ne = {64, 5, 4, 3}, + bool v = false, + float eps = 1e-6f, + bool inplace = false) + : type(type), ne(ne), v(v), eps(eps), inplace(inplace) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + if (v) { + a = ggml_view_4d(ctx, a, a->ne[0]/2, a->ne[1]/2, a->ne[2]/2, a->ne[3]/2, a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(a, "view of a"); + } + + ggml_tensor * out; + if (inplace) { + out = ggml_rms_norm_inplace(ctx, a, eps); + } else { + out = ggml_rms_norm(ctx, a, eps); + } + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -10.f, 10.f); + } + } + + float grad_eps() override { + return 1.0f; + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_RMS_NORM_BACK +struct test_rms_norm_back : public test_case { + const ggml_type type; + const std::array ne; + const float eps; + + std::string vars() override { + return VARS_TO_STR3(type, ne, eps); + } + + test_rms_norm_back(ggml_type type = GGML_TYPE_F32, + std::array ne = {64, 5, 4, 3}, + float eps = 1e-6f) + : type(type), ne(ne), eps(eps) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(b, "b"); + + ggml_tensor * out = ggml_rms_norm_back(ctx, a, b, eps); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -10.f, 10.f); + } + } +}; + +// GGML_OP_RMS_NORM + GGML_OP_MUL + GGML_OP_ADD +struct test_rms_norm_mul_add : public test_case { + const ggml_type type; + const std::array ne; + const float eps; + const bool broadcast; + const bool multi_add; // test a sequence of adds feeding into rms_norm + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "RMS_NORM_MUL_ADD"; + } + + bool run_whole_graph() override { return true; } + + std::string vars() override { + return VARS_TO_STR5(type, ne, eps, broadcast, multi_add); + } + + test_rms_norm_mul_add(ggml_type type = GGML_TYPE_F32, + std::array ne = {64, 5, 4, 3}, + float eps = 1e-6f, bool broadcast = false, bool multi_add = false) + : type(type), ne(ne), eps(eps), broadcast(broadcast), multi_add(multi_add) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + std::array broadcast_dims = {ne[0]*2, ne[1]*3, ne[2]*3, ne[3]*4}; + + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, broadcast ? broadcast_dims.data() : ne.data()); + ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_tensor * c = ggml_new_tensor(ctx, type, 4, ne.data()); + + ggml_set_param(a); + ggml_set_name(a, "a"); + ggml_set_param(b); + ggml_set_name(b, "b"); + ggml_set_param(c); + ggml_set_name(c, "c"); + + // Use a, b and c early, so we don't end up with an OP_NONE between rms_norm and mul + a = ggml_add(ctx, ggml_add(ctx, a, b), c); + if (multi_add) { + a = ggml_add(ctx, ggml_add(ctx, a, b), c); + } + ggml_tensor * out = ggml_add(ctx, ggml_mul(ctx, ggml_rms_norm(ctx, a, eps), b), c); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -10.f, 10.f); + } + } + + float grad_eps() override { + return 1.0f; + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_ADD + GGML_OP_RMS_NORM (fused operation) +struct test_add_rms_norm : public test_case { + const ggml_type type; + const std::array ne; + const float eps; + const bool broadcast; + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "ADD_RMS_NORM"; + } + + bool run_whole_graph() override { return true; } + + std::string vars() override { + return VARS_TO_STR4(type, ne, eps, broadcast); + } + + test_add_rms_norm(ggml_type type = GGML_TYPE_F32, + std::array ne = {64, 5, 4, 3}, + float eps = 1e-6f, bool broadcast = false) + : type(type), ne(ne), eps(eps), broadcast(broadcast) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + std::array broadcast_dims = {ne[0]*2, ne[1]*3, ne[2]*3, ne[3]*4}; + + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, broadcast ? broadcast_dims.data() : ne.data()); + ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); + + ggml_set_param(a); + ggml_set_name(a, "a"); + ggml_set_param(b); + ggml_set_name(b, "b"); + + // ADD operation followed by RMS_NORM + ggml_tensor * add_result = ggml_add(ctx, a, b); + ggml_set_name(add_result, "add_result"); + + ggml_tensor * out = ggml_rms_norm(ctx, add_result, eps); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -10.f, 10.f); + } + } + + float grad_eps() override { + return 1.0f; + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_UNARY(RELU) + GGML_OP_SQR (fused operation) +struct test_relu_sqr : public test_case { + const ggml_type type; + const std::array ne; + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "RELU_SQR"; + } + + bool run_whole_graph() override { return true; } + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_relu_sqr(ggml_type type = GGML_TYPE_F32, + std::array ne = {128, 2, 2, 2}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * r = ggml_relu(ctx, a); + ggml_set_name(r, "relu"); + + ggml_tensor * out = ggml_sqr(ctx, r); + ggml_set_name(out, "out"); + + return out; + } +}; + +// SNAKE activation fusion: y = x + sin(a*x)^2 * inv_b +// CUDA backend matches the naive 5-op chain (mul, sin, sqr, mul, add) +// and dispatches a single fused kernel. +struct test_snake_fuse : public test_case { + const ggml_type type; + const std::array ne; // [T, C, D2, D3] + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "SNAKE_FUSE"; + } + + bool run_whole_graph() override { return true; } + + double max_nmse_err() override { + // BF16 epsilon ~ 7.8e-3, F16 epsilon ~ 9.7e-4: relax tolerance to match + // the natural roundoff drift between the naive CPU chain and the fused + // CUDA kernel. F32 keeps the default tight bound. + switch (type) { + case GGML_TYPE_BF16: return 5e-3; + case GGML_TYPE_F16: return 5e-5; + default: return 1e-7; + } + } + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_snake_fuse(ggml_type type = GGML_TYPE_F32, + std::array ne = {256, 192, 1, 1}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * x = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_name(x, "x"); + + ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, ne[1]); + ggml_set_name(a, "a"); + + ggml_tensor * inv_b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, ne[1]); + ggml_set_name(inv_b, "inv_b"); + + // exact 5-op chain that BigVGAN / Vocos frontends emit + ggml_tensor * ax = ggml_mul(ctx, x, a); + ggml_tensor * sin_ax = ggml_sin(ctx, ax); + ggml_tensor * sin_sq = ggml_sqr(ctx, sin_ax); + ggml_tensor * scaled = ggml_mul(ctx, sin_sq, inv_b); + ggml_tensor * out = ggml_add(ctx, x, scaled); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + // x in [-pi, pi] to exercise sin periodicity, params in default [-1, 1] + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + const std::string name = ggml_get_name(t); + if (name == "x") { + init_tensor_uniform(t, -3.14159f, 3.14159f); + } else { + init_tensor_uniform(t); + } + } + } +}; + +// GGML_OP_SSM_CONV +struct test_ssm_conv : public test_case { + const ggml_type type; + const std::array ne_a; + const std::array ne_b; + + std::string vars() override { + return VARS_TO_STR3(type, ne_a, ne_b); + } + + test_ssm_conv(ggml_type type = GGML_TYPE_F32, + std::array ne_a = {10, 10, 10, 1}, + std::array ne_b = {3, 3, 1, 1}) + : type(type), ne_a(ne_a), ne_b(ne_b) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne_b.data()); + ggml_tensor * out = ggml_ssm_conv(ctx, a, b); + return out; + } +}; + +// GGML_OP_SSM_CONV + GGML_OP_ADD (channel-wise bias, optional) + GGML_OP_UNARY(SILU) (fused operation) +struct test_ssm_conv_bias_silu : public test_case { + const ggml_type type; + const std::array ne_a; + const std::array ne_b; + const bool fuse_bias; + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "SSM_CONV_BIAS_SILU"; + } + + bool run_whole_graph() override { return true; } + + std::string vars() override { + return VARS_TO_STR4(type, ne_a, ne_b, fuse_bias); + } + + test_ssm_conv_bias_silu(ggml_type type, std::array ne_a, std::array ne_b, + bool fuse_bias) + : type(type), ne_a(ne_a), ne_b(ne_b), fuse_bias(fuse_bias) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne_b.data()); + ggml_set_name(a, "a"); + ggml_set_name(b, "b"); + + ggml_tensor * out = ggml_ssm_conv(ctx, a, b); + + if (fuse_bias) { + ggml_tensor * bias = ggml_new_tensor_1d(ctx, type, out->ne[0]); + ggml_set_name(bias, "bias"); + out = ggml_add(ctx, out, bias); + } + + out = ggml_silu(ctx, out); + + ggml_set_name(out, "out"); + return out; + } +}; + +// GGML_OP_SSM_SCAN +struct test_ssm_scan : public test_case { + const ggml_type type; + + const int64_t d_state; + const int64_t head_dim; + const int64_t n_head; + const int64_t n_group; + const int64_t n_seq_tokens; + const int64_t n_seqs; + const bool xbc_overlap; + + std::string vars() override { + return VARS_TO_STR8(type, d_state, head_dim, n_head, n_group, n_seq_tokens, n_seqs, xbc_overlap); + } + + test_ssm_scan(ggml_type type = GGML_TYPE_F32, + int64_t d_state = 32, + int64_t head_dim = 1, // non-zero for Mamba-2 + int64_t n_head = 32, + int64_t n_group = 1, + int64_t n_seq_tokens = 32, + int64_t n_seqs = 32, + bool xbc_overlap = false) + : type(type), d_state(d_state), head_dim(head_dim), n_head(n_head), n_group(n_group), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs), xbc_overlap(xbc_overlap) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * s = ggml_new_tensor_4d(ctx, type, d_state, head_dim, n_head, n_seqs); + ggml_tensor * dt = ggml_new_tensor_3d(ctx, type, n_head, n_seq_tokens, n_seqs); + ggml_tensor * A = ggml_new_tensor_2d(ctx, type, (head_dim > 1) ? 1 : d_state, n_head); + ggml_tensor * x; + ggml_tensor * B; + ggml_tensor * C; + + if (xbc_overlap) { + ggml_tensor * xbc = ggml_new_tensor_4d(ctx, type, d_state, n_head, n_seq_tokens, 2 * n_seqs); + x = ggml_view_4d(ctx, xbc, head_dim, n_head, n_seq_tokens, n_seqs, + xbc->nb[1], xbc->nb[2], xbc->nb[3], xbc->nb[3]); + B = ggml_view_4d(ctx, xbc, d_state, n_group, n_seq_tokens, n_seqs, + xbc->nb[1], xbc->nb[2], xbc->nb[3], 0); + C = ggml_view_4d(ctx, xbc, d_state, n_group, n_seq_tokens, n_seqs, + xbc->nb[1], xbc->nb[2], xbc->nb[3], 2 * xbc->nb[3]); + } else { + x = ggml_new_tensor_4d(ctx, type, head_dim, n_head, n_seq_tokens, n_seqs); + B = ggml_new_tensor_4d(ctx, type, d_state, n_group, n_seq_tokens, n_seqs); + C = ggml_new_tensor_4d(ctx, type, d_state, n_group, n_seq_tokens, n_seqs); + } + ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_seqs); + ggml_tensor * out = ggml_ssm_scan(ctx, s, x, dt, A, B, C, ids); + return out; + } + + // similar to test_mul_mat_id + void initialize_tensors(ggml_context * ctx) override { + std::random_device rd; + std::default_random_engine rng(rd()); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_I32) { + if (ggml_is_view_op(t->op)) { continue; } + // ids + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector data(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + data[i] = i; + } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); + } + } else { + init_tensor_uniform(t); + } + } + } +}; + +// GGML_OP_RWKV_WKV6 +struct test_rwkv_wkv6 : public test_case { + const ggml_type type; + + const int64_t head_count; + const int64_t head_size; + const int64_t n_seq_tokens; + const int64_t n_seqs; + + std::string vars() override { + return VARS_TO_STR5(type, head_count, head_size, n_seq_tokens, n_seqs); + } + + test_rwkv_wkv6(ggml_type type = GGML_TYPE_F32, + int64_t head_count = 32, int64_t head_size = 64, int64_t n_seq_tokens = 32, int64_t n_seqs = 32) + : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t n_tokens = n_seq_tokens * n_seqs; + ggml_tensor * r = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * k = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * v = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * tf = ggml_new_tensor(ctx, type, 2, std::vector{ head_size, head_count }.data()); + ggml_tensor * td = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * s = ggml_new_tensor(ctx, type, 2, std::vector{ head_size * head_size * head_count, n_seqs }.data()); + ggml_tensor * out = ggml_rwkv_wkv6(ctx, k, v, r, tf, td, s); + return out; + } +}; + +// GGML_OP_GATED_DELTA_NET +struct test_gated_delta_net : public test_case { + const ggml_type type; + + const int64_t head_count; + const int64_t head_size; + const int64_t n_seq_tokens; + const int64_t n_seqs; + const int v_repeat; + const bool permuted; + const bool kda; + const int64_t K; // snapshot slot count: 1 = final-only, >1 = last K states + + std::string vars() override { + return VARS_TO_STR9(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K); + } + + test_gated_delta_net(ggml_type type = GGML_TYPE_F32, + int64_t head_count = 4, int64_t head_size = 16, int64_t n_seq_tokens = 1, int64_t n_seqs = 1, + int v_repeat = 1, bool permuted = false, bool kda = false, int64_t K = 1) + : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs), + v_repeat(v_repeat), permuted(permuted), kda(kda), K(K) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * q; + ggml_tensor * k; + ggml_tensor * v; + if (permuted) { + // create with dims 1 and 2 swapped, then permute back to get non-contiguous layout + q = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs), 0, 2, 1, 3); + k = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs), 0, 2, 1, 3); + v = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count * v_repeat, n_seqs), 0, 2, 1, 3); + } else { + q = ggml_new_tensor_4d(ctx, type, head_size, head_count, n_seq_tokens, n_seqs); + k = ggml_new_tensor_4d(ctx, type, head_size, head_count, n_seq_tokens, n_seqs); + v = ggml_new_tensor_4d(ctx, type, head_size, head_count * v_repeat, n_seq_tokens, n_seqs); + } + ggml_set_name(q, "q"); + ggml_set_name(k, "k"); + ggml_set_name(v, "v"); + const int64_t g_ne0 = kda ? head_size : 1; + ggml_tensor * g = ggml_new_tensor_4d(ctx, type, g_ne0, head_count * v_repeat, n_seq_tokens, n_seqs); + ggml_tensor * beta = ggml_new_tensor_4d(ctx, type, 1, head_count * v_repeat, n_seq_tokens, n_seqs); + ggml_tensor * state = ggml_new_tensor_4d(ctx, type, head_size, head_size, head_count * v_repeat, n_seqs); + ggml_set_name(g, "g"); + ggml_set_name(beta, "beta"); + ggml_set_name(state, "state"); + // q/k are L2-normalised in qwen35/kimi-linear before delta_net + q = ggml_l2_norm(ctx, q, 1e-6f); + k = ggml_l2_norm(ctx, k, 1e-6f); + ggml_tensor * out = ggml_gated_delta_net(ctx, q, k, v, g, beta, state, K); + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (ggml_is_view_op(t->op)) { continue; } + if (strcmp(t->name, "g") == 0) { + init_tensor_uniform(t, -20.0f, -1e-4f); + } else if (strcmp(t->name, "beta") == 0) { + init_tensor_uniform(t, 0.0f, 1.0f); + } else if (strcmp(t->name, "v") == 0) { + init_tensor_uniform(t, -0.3f, 5.0f); + } else { + init_tensor_uniform(t); + } + } + } +}; + +// GGML_OP_GATED_LINEAR_ATTN +struct test_gla : public test_case { + const ggml_type type; + + const int64_t head_count; + const int64_t head_size; + const int64_t n_seq_tokens; + const int64_t n_seqs; + + std::string vars() override { + return VARS_TO_STR5(type, head_count, head_size, n_seq_tokens, n_seqs); + } + + test_gla(ggml_type type = GGML_TYPE_F32, + int64_t head_count = 32, int64_t head_size = 64, int64_t n_seq_tokens = 32, int64_t n_seqs = 32) + : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t n_tokens = n_seq_tokens * n_seqs; + ggml_tensor * q = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * k = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * v = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * g = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * s = ggml_new_tensor(ctx, type, 2, std::vector{ head_size * head_size * head_count, n_seqs }.data()); + ggml_tensor * out = ggml_gated_linear_attn(ctx, k, v, q, g, s, pow(head_size, -0.5)); + return out; + } +}; + +// GGML_OP_RWKV_WKV7 +struct test_rwkv_wkv7 : public test_case { + const ggml_type type; + + const int64_t head_count; + const int64_t head_size; + const int64_t n_seq_tokens; + const int64_t n_seqs; + + std::string vars() override { + return VARS_TO_STR5(type, head_count, head_size, n_seq_tokens, n_seqs); + } + + test_rwkv_wkv7(ggml_type type = GGML_TYPE_F32, + int64_t head_count = 32, int64_t head_size = 64, int64_t n_seq_tokens = 32, int64_t n_seqs = 32) + : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t n_tokens = n_seq_tokens * n_seqs; + ggml_tensor * r = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * w = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * k = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * v = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * a = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + ggml_tensor * b = ggml_new_tensor(ctx, type, 3, std::vector{ head_size, head_count, n_tokens }.data()); + // Outputs may become NaN with long seqlen without these normalization + a = ggml_l2_norm(ctx, a, 1e-7F); + b = ggml_l2_norm(ctx, b, 1e-7F); + ggml_tensor * s = ggml_new_tensor(ctx, type, 2, std::vector{ head_size * head_size * head_count, n_seqs }.data()); + ggml_tensor * out = ggml_rwkv_wkv7(ctx, r, w, k, v, a, b, s); + return out; + } +}; + +// GGML_OP_MUL_MAT +struct test_mul_mat : public test_case { + const ggml_type type_a; + const ggml_type type_b; + const int64_t m; + const int64_t n; + const int64_t k; + const std::array bs; // dims 3 and 4 + const std::array nr; // repeat in dims 3 and 4 + const std::array per; // permutation of dimensions + const int64_t k_v; // size of k in memory, resulting in a non-contiguous view for k_v > k, no view for k_v == 0 + const uint32_t o; // number of outputs + + std::string vars() override { + return VARS_TO_STR10(type_a, type_b, m, n, k, bs, nr, per, k_v, o); + } + + double max_nmse_err() override { + return 5e-4; + } + + double max_nmse_err(ggml_backend_t backend) override { + // for blackwell we quantize activations to mxfp4 instead of q8_1 so we add higher tolerance + if ((type_a == GGML_TYPE_MXFP4 || type_a == GGML_TYPE_NVFP4) && backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) { + return 2e-2; + } + return max_nmse_err(); + } + + int64_t grad_nmax() override { + return 20000; + } + + uint64_t op_flops(ggml_tensor * t) override { + GGML_UNUSED(t); + return 2 * m * n * k * bs[0] * nr[0] * bs[1] * nr[1]; + } + + test_mul_mat(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, + int64_t m = 32, int64_t n = 32, int64_t k = 32, + std::array bs = {10, 10}, + std::array nr = {2, 2}, + std::array per = {0, 1, 2, 3}, + int64_t k_v = 0, uint32_t o = 1) + : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), per(per), k_v(k_v), o(o) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + // C^T = A * B^T: (k, m) * (k, n) => (m, n) + ggml_tensor * a; + ggml_tensor * b; + + const int npermuted = (per[0] != 0) + (per[1] != 1) + (per[2] != 2) + (per[3] != 3); + if (npermuted > 0) { + GGML_ASSERT(npermuted == 2); + GGML_ASSERT(k_v == 0); // not handled + GGML_ASSERT(!ggml_is_quantized(type_a) || per[0] == 0); + GGML_ASSERT(!ggml_is_quantized(type_b) || per[0] == 0); + + // Create tensors with the permuted dimensions, then permute them back to the dimensions given by m,n,k. + const int64_t ne_a[4] = {k, m, bs[0], bs[1]}; + const int64_t ne_b[4] = {k, n, bs[0]*nr[0], bs[1]*nr[1]}; + + a = ggml_new_tensor_4d(ctx, type_a, ne_a[per[0]], ne_a[per[1]], ne_a[per[2]], ne_a[per[3]]); + b = ggml_new_tensor_4d(ctx, type_b, ne_b[per[0]], ne_b[per[1]], ne_b[per[2]], ne_b[per[3]]); + if (!ggml_is_quantized(type_a)) { + if (bs[1] == 1 && nr[1] == 1) { + ggml_set_param(a); + } + ggml_set_param(b); + } + ggml_set_name(a, "a"); + ggml_set_name(b, "b"); + + a = ggml_permute(ctx, a, per[0], per[1], per[2], per[3]); + b = ggml_permute(ctx, b, per[0], per[1], per[2], per[3]); + ggml_set_name(a, "a_permuted"); + ggml_set_name(b, "b_permuted"); + } else { + const int64_t k_physical = k_v == 0 ? k : k_v; + a = ggml_new_tensor_4d(ctx, type_a, k_physical, m, bs[0], bs[1]); + b = ggml_new_tensor_4d(ctx, type_b, k_physical, n, bs[0]*nr[0], bs[1]*nr[1]); + + if (!ggml_is_quantized(type_a)) { + if (bs[1] == 1 && nr[1] == 1) { + ggml_set_param(a); + } + ggml_set_param(b); + } + + if (k_v != 0) { + GGML_ASSERT(k_v > k); + a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], a->nb[2], a->nb[3], 0); + b = ggml_view_4d(ctx, b, k, n, bs[0]*nr[0], bs[1]*nr[1], b->nb[1], b->nb[2], b->nb[3], 0); + } + ggml_set_name(a, "a"); + ggml_set_name(b, "b"); + } + + ggml_tensor * out = ggml_mul_mat(ctx, a, b); + ggml_set_name(out, "out"); + for (uint32_t i = 1; i < o; ++i) { + ggml_tensor * out2 = ggml_mul_mat(ctx, a, b); + ggml_set_name(out2, "out2"); + out = ggml_add(ctx, out, out2); + } + + return out; + } + + bool run_whole_graph() override { return o > 1; } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return ggml_op_name(GGML_OP_MUL_MAT); + } +}; + +// GGML_HINT_SRC0_IS_HADAMARD +struct test_mul_mat_hadamard : public test_mul_mat { + test_mul_mat_hadamard(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, + int64_t m = 32, int64_t n = 32, int64_t k = 32, + std::array bs = {1, 1}, + std::array nr = {1, 1}) + : test_mul_mat(type_a, type_b, m, n, k, bs, nr) { + GGML_ASSERT(type_a == GGML_TYPE_F32); + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * out = test_mul_mat::build_graph(ctx); + // Find the mul_mat op in the graph and set the hint + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->op == GGML_OP_MUL_MAT) { + ggml_mul_mat_set_hint(t, GGML_HINT_SRC0_IS_HADAMARD); + } + } + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "a") == 0) { + const int64_t n_cols = t->ne[0]; + const int64_t n_rows = ggml_nrows(t); + std::vector data(n_cols * n_rows); + float scale = 1.0f / sqrtf((float)n_cols); + for (int64_t r = 0; r < n_rows; r++) { + float * row_data = data.data() + r * n_cols; + for (int64_t i = 0; i < n_cols; i++) { + int pop = 0; + int64_t val = r & i; + while (val) { + pop += (val & 1); + val >>= 1; + } + row_data[i] = (pop % 2 == 0) ? scale : -scale; + } + } + ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(float)); + } else if (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_F16) { + init_tensor_uniform(t); + } + } + } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MUL_MAT_HADAMARD"; + } +}; + +static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats) { + std::random_device rd; + std::default_random_engine rng(rd()); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_I32) { + if (ggml_is_view_op(t->op)) { continue; } + // ids + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector data(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + data[i] = i % n_mats; + } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); + } + } else { + init_tensor_uniform(t); + } + } +} + +// GGML_OP_MUL_MAT_ID +struct test_mul_mat_id : public test_case { + const ggml_type type_a; + const ggml_type type_b; + const int n_mats; + const int n_used; + const bool b; // broadcast b matrix + const int64_t m; + const int64_t n; + const int64_t k; + + std::string vars() override { + return VARS_TO_STR8(type_a, type_b, n_mats, n_used, b, m, n, k); + } + + double max_nmse_err() override { + return 5e-4; + } + + double max_nmse_err(ggml_backend_t backend) override { + // for blackwell we quantize activations to mxfp4 instead of q8_1 so we add higher tolerance + if ((type_a == GGML_TYPE_MXFP4 || type_a == GGML_TYPE_NVFP4) && backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) { + return 2e-2; + } + return max_nmse_err(); + } + + uint64_t op_flops(ggml_tensor * t) override { + GGML_UNUSED(t); + return 2 * m * k * n * n_used; + } + + test_mul_mat_id(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, + int n_mats = 8, int n_used = 2, bool b = false, + int64_t m = 32, int64_t n = 32, int64_t k = 32) + : type_a(type_a), type_b(type_b), n_mats(n_mats), n_used(n_used), b(b), + m(m), n(n), k(k) { + GGML_ASSERT(n_used <= n_mats); + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + // C^T = A * B^T: (k, m) * (k, n) => (m, n) + ggml_tensor * as = ggml_new_tensor_3d(ctx, type_a, k, m, n_mats); + ggml_set_name(as, "as"); + + ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n); + ggml_set_name(ids, "ids"); + if (n_used != n_mats) { + ids = ggml_view_2d(ctx, ids, n_used, n, ids->nb[1], 0); + ggml_set_name(ids, "view_of_ids"); + } + + ggml_tensor * b = ggml_new_tensor_3d(ctx, type_b, k, this->b ? 1 : n_used, n); + ggml_set_name(b, "b"); + + ggml_tensor * out = ggml_mul_mat_id(ctx, as, b, ids); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + init_mul_mat_id_tensors(ctx, n_mats); + } +}; + +// GGML_OP_MUL_MAT_ID + GGML_OP_ADD or GGML_OP_MUL +struct test_mul_mat_id_fusion : public test_case { + const ggml_type type_a; + const ggml_type type_b; + const int n_mats; + const int n_used; + const bool b; // broadcast b matrix + const int64_t m; + const int64_t n; + const int64_t k; + const uint32_t o; // number of outputs + const bool mul; + + std::string vars() override { + return VARS_TO_STR10(type_a, type_b, n_mats, n_used, b, m, n, k, o, mul); + } + + double max_nmse_err() override { + return 5e-4; + } + + uint64_t op_flops(ggml_tensor * t) override { + GGML_UNUSED(t); + return 2 * m * k * n * n_used; + } + + test_mul_mat_id_fusion(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, + int n_mats = 8, int n_used = 2, bool b = false, + int64_t m = 32, int64_t n = 32, int64_t k = 32, uint32_t o = 1, bool mul = false) + : type_a(type_a), type_b(type_b), n_mats(n_mats), n_used(n_used), b(b), + m(m), n(n), k(k), o(o), mul(mul) { + GGML_ASSERT(n_used <= n_mats); + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + // C^T = A * B^T: (k, m) * (k, n) => (m, n) + ggml_tensor * as = ggml_new_tensor_3d(ctx, type_a, k, m, n_mats); + ggml_set_name(as, "as"); + + ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n); + ggml_set_name(ids, "ids"); + if (n_used != n_mats) { + ids = ggml_view_2d(ctx, ids, n_used, n, ids->nb[1], 0); + ggml_set_name(ids, "view_of_ids"); + } + + ggml_tensor * b = ggml_new_tensor_3d(ctx, type_b, k, this->b ? 1 : n_used, n); + ggml_set_name(b, "b"); + + ggml_tensor * out = ggml_mul_mat_id(ctx, as, b, ids); + ggml_set_name(out, "out"); + + for (uint32_t i = 1; i < o; ++i) { + ggml_tensor * a2 = ggml_new_tensor_3d(ctx, type_a, k, m, n_mats); + ggml_tensor * out2 = ggml_mul_mat_id(ctx, a2, b, ids); + ggml_set_name(out2, "out2"); + out = ggml_add(ctx, out, out2); + } + + if (mul) { + std::array ne { 1, out->ne[1], out->ne[2], out->ne[3] }; + ne[0] = 1; + ggml_tensor * m = ggml_new_tensor(ctx, out->type, 4, ne.data()); + out = ggml_mul(ctx, out, m); + } + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + init_mul_mat_id_tensors(ctx, n_mats); + } + + bool run_whole_graph() override { return true; } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MUL_MAT_ID_FUSION"; + } +}; + +// GGML_OP_OUT_PROD +struct test_out_prod : public test_case { + const ggml_type type_a; + const ggml_type type_b; + const int64_t m; + const int64_t n; + const int64_t k; + const std::array bs; // dims 3 and 4 + const std::array nr; // repeat in dims 3 and 4 + const bool trans_b; + + std::string vars() override { + return VARS_TO_STR8(type_a, type_b, m, n, k, bs, nr, trans_b); + } + + double max_nmse_err() override { + return 5e-4; + } + + test_out_prod(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, + int64_t m = 32, int64_t n = 32, int64_t k = 32, + std::array bs = {10, 10}, + std::array nr = {2, 2}, + bool trans_b = false) + : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), trans_b(trans_b) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_4d(ctx, type_a, m, k, bs[0], bs[1]); + ggml_set_name(a, "a"); + + ggml_tensor * b; + if (trans_b) { + b = ggml_new_tensor_4d(ctx, type_b, k, n, bs[0]*nr[0], bs[1]*nr[1]); + b = ggml_transpose(ctx, b); + } else { + b = ggml_new_tensor_4d(ctx, type_b, n, k, bs[0]*nr[0], bs[1]*nr[1]); + } + ggml_set_name(b, "b"); + + ggml_tensor * out = ggml_out_prod(ctx, a, b); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_SQR +struct test_sqr : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_sqr(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 5, 4, 3}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_sqr(ctx, a); + ggml_set_name(out, "out"); + + return out; + } + + float grad_eps() override { + return 0.1f * 0.25f*ne[0]*ne[1]*ne[2]*ne[3]; // 10% of expected value of sum. + } +}; + +// GGML_OP_SQRT +struct test_sqrt : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_sqrt(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 3, 3, 2}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_sqrt(ctx, a); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + // fill with positive values + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, 50.0f, 100.0f); + } + } + + float grad_eps() override { + return 20.0f; + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_LOG +struct test_log : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_log(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 5, 4, 3}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_log(ctx, a); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + // log(1) == 0, cluster values there to keep the sum low for better precision in the backward pass: + init_tensor_uniform(t, 0.9f, 1.1f); + } + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_SIN +struct test_sin : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_sin(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 2, 2, 2}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_sin(ctx, a); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -6.5f, 6.5f); // Covers interval [-2*pi, 2*pi]. + } + } + + double max_maa_err() override { + return 1e-3; + } + + float grad_eps() override { + return 0.2f; + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_COS +struct test_cos : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_cos(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 2, 2, 2}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_cos(ctx, a); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -6.5f, 6.5f); // Covers interval [-2*pi, 2*pi]. + } + } + + double max_maa_err() override { + return 1e-3; + } + + float grad_eps() override { + return 0.2f; + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_CLAMP +struct test_clamp : public test_case { + const ggml_type type; + const std::array ne; + float min; + float max; + + std::string vars() override { + return VARS_TO_STR4(type, ne, min, max); + } + + test_clamp(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 5, 4, 3}, + float min = -0.5f, float max = 0.5f) + : type(type), ne(ne), min(min), max(max) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_clamp(ctx, a, min, max); + ggml_set_name(out, "out"); + + return out; + } + + float grad_eps() override { + return 1e-2f; + } + + std::vector grad_expect() override { + return {0.0f, 1.0f}; + } +}; + +// GGML_OP_FLOOR +struct test_floor : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_floor(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 2, 2, 2}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_floor(ctx, a); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -10.0f, 10.0f); + } + } +}; + +// GGML_OP_CEIL +struct test_ceil : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_ceil(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 2, 2, 2}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_ceil(ctx, a); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -10.0f, 10.0f); + } + } +}; + +// GGML_OP_ROUND +struct test_round : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_round(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 2, 2, 2}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_round(ctx, a); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -10.0f, 10.0f); + } + } +}; + +// GGML_OP_TRUNC +struct test_trunc : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_trunc(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 2, 2, 2}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_trunc(ctx, a); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -10.0f, 10.0f); + } + } +}; + +// GGML_OP_DIAG_MASK_INF +struct test_diag_mask_inf : public test_case { + const ggml_type type; + const std::array ne; + const int n_past; + + std::string vars() override { + return VARS_TO_STR3(type, ne, n_past); + } + + test_diag_mask_inf(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 10, 3, 2}, + int n_past = 5) + : type(type), ne(ne), n_past(n_past) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_diag_mask_inf(ctx, a, n_past); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_SOFT_MAX +struct test_soft_max : public test_case { + const ggml_type type; + const std::array ne; + const bool mask; + const bool sinks; + const ggml_type m_prec; + const std::array nr23; // broadcast only dims 2 and 3 + const float scale; + const float max_bias; + const bool inplace; + + std::string vars() override { + return VARS_TO_STR9(type, ne, mask, sinks, m_prec, nr23, scale, max_bias, inplace); + } + + // the 1024 test with bias occasionally fails: + // SOFT_MAX(type=f32,ne=[1024,16,1,1],mask=1,scale=1.000000,max_bias=8.000000): [SOFT_MAX] NMSE = 0.000000103 > 0.000000100 FAIL + virtual double max_nmse_err() override { + return 1e-6; + } + + test_soft_max(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 5, 4, 3}, + bool mask = false, + bool sinks = false, + ggml_type m_prec = GGML_TYPE_F32, + std::array nr23 = {1, 1}, + float scale = 1.0f, + float max_bias = 0.0f, + bool inplace = false) + : type(type), ne(ne), mask(mask), sinks(sinks), m_prec(m_prec), nr23(nr23), scale(scale), max_bias(max_bias), inplace(inplace) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2]*nr23[0], ne[3]*nr23[1]); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * mask = nullptr; + if (this->mask) { + mask = ggml_new_tensor_4d(ctx, m_prec, ne[0], ne[1], ne[2], ne[3]); + ggml_set_name(mask, "mask"); + } + + ggml_tensor * sinks = nullptr; + if (this->sinks) { + sinks = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[2]*nr23[0]); + ggml_set_name(sinks, "sinks"); + } + + ggml_tensor * out; + if (inplace) { + out = ggml_soft_max_ext_inplace(ctx, a, mask, scale, max_bias); + } else { + out = ggml_soft_max_ext(ctx, a, mask, scale, max_bias); + } + ggml_soft_max_add_sinks(out, sinks); + ggml_set_name(out, "out"); + + return out; + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_SOFT_MAX_BACK +struct test_soft_max_back : public test_case { + const ggml_type type; + const std::array ne; + const float scale; + const float max_bias; + + std::string vars() override { + return VARS_TO_STR4(type, ne, scale, max_bias); + } + + test_soft_max_back(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 5, 4, 3}, + float scale = 1.0f, + float max_bias = 0.0f) + : type(type), ne(ne), scale(scale), max_bias(max_bias) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_soft_max_ext_back(ctx, a, b, scale, max_bias); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_ROPE + GGML_OP_ROPE_BACK +struct test_rope : public test_case { + const ggml_type type; + const std::array ne_a; + int n_dims; + int mode; + int n_ctx; // used to generate positions + float fs; // freq_scale + float ef; // ext_factor + float af; // attn_factor + bool ff; + int v; // view (1 : non-contiguous a) + bool forward; + bool inplace; + + std::string vars() override { + // forward can be inferred from the op, does not need to be printed + return VARS_TO_STR11(type, ne_a, n_dims, mode, n_ctx, fs, ef, af, ff, v, inplace); + } + + test_rope(ggml_type type = GGML_TYPE_F32, + std::array ne_a = {10, 5, 3, 1}, + int n_dims = 10, int mode = GGML_ROPE_TYPE_NORMAL, int n_ctx = 512, float fs = 1.0f, + float ef = 0.0f, float af = 0.0f, bool ff = false, int v = 0, bool forward = true, bool inplace = false) + : type(type), ne_a(ne_a), n_dims(n_dims), mode(mode), n_ctx(n_ctx), fs(fs), ef(ef), af(af), ff(ff), v(v), forward(forward), inplace(inplace) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a; + if (v & 1) { + auto ne = ne_a; ne[0] *= 2; ne[1] *= 4; ne[2] *= 3; + a = ggml_new_tensor(ctx, type, 4, ne.data()); + if (forward) { + ggml_set_param(a); + } + ggml_set_name(a, "a"); + + a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(a, "view_of_a"); + } else if (v == 2) { + // second-half slice along dim 0 (mimics build_rope_2d in clip.cpp). + // The non-zero view offset (ne_a[0] * elem_size) often produces a + // non-aligned buffer offset, which exercises backends' alignment paths. + auto ne = ne_a; ne[0] *= 2; + a = ggml_new_tensor(ctx, type, 4, ne.data()); + if (forward) { + ggml_set_param(a); + } + ggml_set_name(a, "a"); + + a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], + a->nb[1], a->nb[2], a->nb[3], + ne_a[0] * ggml_element_size(a)); + ggml_set_name(a, "view_of_a"); + } else { + a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + if (forward) { + ggml_set_param(a); + } + ggml_set_name(a, "a"); + } + + const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; + const bool is_vision = mode == GGML_ROPE_TYPE_VISION; + + ggml_tensor * pos; + if (is_mrope || is_vision) { + pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne_a[2] * 4); + } else { + pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne_a[2]); + } + ggml_set_name(pos, "pos"); + + ggml_tensor * freq = nullptr; + if (ff) { + freq = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_dims/2); + ggml_set_name(freq, "freq"); + } + + ggml_tensor * out; + if (is_mrope) { + if (is_vision) { + GGML_ASSERT(n_dims/4 > 0); + int rope_sections[4] = {n_dims/4, n_dims/4, 0, 0}; // Vision-RoPE only use first two dimension for image (x, y) coordinate + if (forward) { + if (inplace) { + out = ggml_rope_multi_inplace(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + } else { + out = ggml_rope_multi(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + } + } else { + out = ggml_rope_multi_back(ctx, a, pos, freq, n_dims/2, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + } + } else { + GGML_ASSERT(n_dims/3 > 0); + int rope_sections[4] = {n_dims/3, n_dims/3, n_dims/3, 0}; + if (forward) { + if (inplace) { + out = ggml_rope_multi_inplace(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + } else { + out = ggml_rope_multi(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + } + } else { + out = ggml_rope_multi_back(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + } + } + } else { + if (forward) { + if (inplace) { + out = ggml_rope_ext_inplace(ctx, a, pos, freq, n_dims, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + } else { + out = ggml_rope_ext(ctx, a, pos, freq, n_dims, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + } + } else { + out = ggml_rope_ext_back(ctx, a, pos, freq, n_dims, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); + } + } + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_I32) { + // pos + const int num_pos_ids = (mode & GGML_ROPE_TYPE_MROPE) ? ne_a[2] * 4 : ne_a[2]; + std::vector data(num_pos_ids); + for (int i = 0; i < num_pos_ids; i++) { + data[i] = rand() % n_ctx; + } + ggml_backend_tensor_set(t, data.data(), 0, num_pos_ids * sizeof(int)); + } else { + if (t->ne[0] == n_dims/2) { + // frequency factors in the range [0.9f, 1.1f] + init_tensor_uniform(t, 0.9f, 1.1f); + } else { + init_tensor_uniform(t); + } + } + } + } + + double max_maa_err() override { + return 1e-3; + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_POOL2D +struct test_pool2d : public test_case { + enum ggml_op_pool pool_type; + const ggml_type type_input; + const std::array ne_input; + // kernel size + const int k0; + const int k1; + // stride + const int s0; + const int s1; + // padding + const int p0; + const int p1; + + std::string vars() override { + return VARS_TO_STR9(pool_type, type_input, ne_input, k0, k1, s0, s1, p0, p1); + } + + test_pool2d(ggml_op_pool pool_type = GGML_OP_POOL_AVG, + ggml_type type_input = GGML_TYPE_F32, + std::array ne_input = {10, 10, 3, 1}, // [input_width, input_height, input_channels, 1] + int k0 = 3, int k1 = 3, + int s0 = 1, int s1 = 1, + int p0 = 1, int p1 = 1) + : pool_type(pool_type), type_input(type_input), ne_input(ne_input), k0(k0), k1(k1), s0(s0), s1(s1), p0(p0), p1(p1) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * input = ggml_new_tensor(ctx, type_input, 4, ne_input.data()); + ggml_set_param(input); + ggml_set_name(input, "input"); + + ggml_tensor * out = ggml_pool_2d(ctx, input, pool_type, k0, k1, s0, s1, p0, p1); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_POOL1D +struct test_pool1d : public test_case { + enum ggml_op_pool pool_type; + const ggml_type type_input; + const std::array ne_input; + const int k0; + const int s0; + const int p0; + + std::string vars() override { + return VARS_TO_STR6(pool_type, type_input, ne_input, k0, s0, p0); + } + + test_pool1d(ggml_op_pool pool_type = GGML_OP_POOL_AVG, + ggml_type type_input = GGML_TYPE_F32, + std::array ne_input = {10, 1, 1, 1}, + int k0 = 3, int s0 = 3, int p0 = 0) + : pool_type(pool_type), type_input(type_input), ne_input(ne_input), k0(k0), s0(s0), p0(p0) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * input = ggml_new_tensor(ctx, type_input, 4, ne_input.data()); + ggml_set_param(input); + ggml_set_name(input, "input"); + + ggml_tensor * out = ggml_pool_1d(ctx, input, pool_type, k0, s0, p0); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_CONV_TRANSPOSE_1D +struct test_conv_transpose_1d : public test_case { + const std::array ne_input; + const std::array ne_kernel; + + const int s0; // stride + const int p0; // padding + const int d0; // dilation + + std::string vars() override { + return VARS_TO_STR5(ne_input, ne_kernel, s0, p0, d0); + } + + test_conv_transpose_1d(std::array ne_input = {197, 32, 1, 1}, // [input_width, input_channels, 1 /* assert in cpu kernel*/, 1 (should be batch)] + std::array ne_kernel = {16, 32, 32, 1}, // [kernel_width, output_channels, input_channels, 1 (should be batch)] + int s0 = 1, int p0 = 0, int d0 = 1) + : ne_input(ne_input), ne_kernel(ne_kernel), s0(s0), p0(p0), d0(d0) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * input = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_input.data()); + ggml_set_name(input, "input"); + + ggml_tensor * kernel = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_kernel.data()); + ggml_set_name(kernel, "kernel"); + + ggml_tensor * out = ggml_conv_transpose_1d(ctx, kernel, input, s0, p0, d0); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_COL2IM_1D +struct test_col2im_1d : public test_case { + const ggml_type type; + const int64_t K; // kernel size + const int64_t OC; // output channels + const int64_t T_in; // input length (number of columns) + const int s0; // stride + const int p0; // padding cropped from both sides + + std::string vars() override { + return VARS_TO_STR6(type, K, OC, T_in, s0, p0); + } + + double max_nmse_err() override { + return type == GGML_TYPE_F32 ? 1e-7 : 5e-4; + } + + test_col2im_1d(ggml_type type = GGML_TYPE_F32, + int64_t K = 4, int64_t OC = 3, int64_t T_in = 7, + int s0 = 2, int p0 = 0) + : type(type), K(K), OC(OC), T_in(T_in), s0(s0), p0(p0) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * cols = ggml_new_tensor_2d(ctx, type, K*OC, T_in); + ggml_set_name(cols, "cols"); + + ggml_tensor * out = ggml_col2im_1d(ctx, cols, s0, (int) OC, p0); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_CONV_TRANSPOSE_2D +struct test_conv_transpose_2d : public test_case { + // Dimensions + const std::array ne_input; + const std::array ne_kernel; + const int stride; + // Types + const ggml_type kernel_type; + + std::string vars() override { + return VARS_TO_STR4(kernel_type, ne_input, ne_kernel, stride); + } + + double max_nmse_err() override { + return 5e-4; // The default 1e-7 is too small for Vulkan. + } + + test_conv_transpose_2d( + std::array ne_input = {10, 10, 3, 1}, // [input_width, input_height, input_channels, 1] + std::array ne_kernel = {3, 3, 3, 1}, // [kernel_width, kernel_height, input_channels, 1] + int stride = 1, + ggml_type kernel_type = GGML_TYPE_F16 + ) : ne_input(ne_input), ne_kernel(ne_kernel), stride(stride), kernel_type(kernel_type) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * input = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_input.data()); + ggml_set_name(input, "input"); + + ggml_tensor * kernel = ggml_new_tensor(ctx, kernel_type, 4, ne_kernel.data()); + ggml_set_name(kernel, "kernel"); + + ggml_tensor * out = ggml_conv_transpose_2d_p0(ctx, kernel, input, stride); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_IM2COL +struct test_im2col : public test_case { + const ggml_type type_input; + const ggml_type type_kernel; + const ggml_type dst_type; + const std::array ne_input; + const std::array ne_kernel; + // stride + const int s0; + const int s1; + // padding + const int p0; + const int p1; + // dilation + const int d0; + const int d1; + // mode + const bool is_2D; + + std::string vars() override { + return VARS_TO_STR12(type_input, type_kernel, dst_type, ne_input, ne_kernel, s0, s1, p0, p1, d0, d1, is_2D); + } + + test_im2col(ggml_type type_input = GGML_TYPE_F32, ggml_type type_kernel = GGML_TYPE_F16, ggml_type dst_type = GGML_TYPE_F32, + std::array ne_input = {10, 10, 3, 1}, // [input_width, input_height, input_channels, 1] + std::array ne_kernel = {3, 3, 3, 1}, // [kernel_width, kernel_height, input_channels, 1] + int s0 = 1, int s1 = 1, + int p0 = 1, int p1 = 1, + int d0 = 1, int d1 = 1, + bool is_2D = true) + : type_input(type_input), type_kernel(type_kernel), dst_type(dst_type), ne_input(ne_input), ne_kernel(ne_kernel), s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), is_2D(is_2D) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * input = ggml_new_tensor(ctx, type_input, 4, ne_input.data()); + ggml_set_param(input); + ggml_set_name(input, "input"); + + ggml_tensor * kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel.data()); + ggml_set_name(kernel, "kernel"); + + ggml_tensor * out = ggml_im2col(ctx, kernel, input, s0, s1, p0, p1, d0, d1, is_2D, dst_type); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_IM2COL_3D +struct test_im2col_3d : public test_case { + const ggml_type type_input; + const ggml_type type_kernel; + const ggml_type dst_type; + const std::array ne_input; + const std::array ne_kernel; + // stride + const int s0; + const int s1; + const int s2; + // padding + const int p0; + const int p1; + const int p2; + // dilation + const int d0; + const int d1; + const int d2; + + const int64_t IC; + const bool v; + + std::string vars() override { + return VARS_TO_STR16(type_input, type_kernel, dst_type, ne_input, ne_kernel, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, v); + } + + test_im2col_3d(ggml_type type_input = GGML_TYPE_F32, ggml_type type_kernel = GGML_TYPE_F16, ggml_type dst_type = GGML_TYPE_F32, + std::array ne_input = {10, 10, 10, 9}, // [OC*IC, KD, KH, KW] + std::array ne_kernel = {3, 3, 3, 1}, // [N*IC, ID, IH, IW] + int64_t IC = 3, + int s0 = 1, int s1 = 1, int s2 = 1, + int p0 = 1, int p1 = 1, int p2 = 1, + int d0 = 1, int d1 = 1, int d2 = 1, + bool v = false) + : type_input(type_input), type_kernel(type_kernel), dst_type(dst_type), ne_input(ne_input), ne_kernel(ne_kernel), s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), IC(IC), v(v) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * input = ggml_new_tensor(ctx, type_input, 4, ne_input.data()); + ggml_set_param(input); + ggml_set_name(input, "input"); + + if (v) { + input = ggml_view_4d(ctx, input, ne_input[0] - 2, ne_input[1] - 2, ne_input[2] - 2, ne_input[3] - 2, input->nb[1], input->nb[2], input->nb[3], 0); + ggml_set_name(input, "view_of_input"); + } + + ggml_tensor * kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel.data()); + ggml_set_name(kernel, "kernel"); + + ggml_tensor * out = ggml_im2col_3d(ctx, kernel, input, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, dst_type); + ggml_set_name(out, "out"); + + return out; + } +}; + +// CONV_2D +struct test_conv_2d : public test_case { + const std::array ne_input; + const std::array ne_kernel; + const ggml_type type_kernel; + const int stride0; + const int stride1; + const int padding0; + const int padding1; + const int dilation0; + const int dilation1; + // Whether the inputs are contiguous in the channel dim or the width dim + const bool cwhn; + + // If true, the direct CONV_2D will be used in the graph, otherwise it + // uses ggml_conv_2d: + // * if the program is called with -o CONV_2D_DIRECT_IMPL, the + // CONV_2D graph will be built, while + // * if the program is called with -o CONV_2D_INDIRECT_IMPL, the + // IM2COL -> MUL_MM graph will be built. + + std::string vars() override { + return VARS_TO_STR10(ne_input, ne_kernel, type_kernel, stride0, stride1, padding0, padding1, dilation0, dilation1, cwhn); + } + + double max_nmse_err() override { + return 5e-4; + } + + uint64_t op_flops(ggml_tensor * t) override { + GGML_UNUSED(t); + // Just counting matmul costs: + // KxCRS @ CRSxNPQ = KxNPQ --> KxNPQx(CRS+CRS-1) flops + + // Copied from ggml.c: int64_t ggml_calc_conv_output_size(int64_t ins, int64_t ks, int s, int p, int d) + auto calc_conv_output_size = [](int64_t ins, int64_t ks, int s, int p, int d) -> int64_t { + return (ins + 2 * p - d * (ks - 1) - 1) / s + 1; + }; + + int64_t W = ne_input[0]; + int64_t H = ne_input[1]; + int64_t KW = ne_kernel[0]; + int64_t KH = ne_kernel[1]; + int64_t Cin = ne_kernel[2]; + int64_t Cout = ne_kernel[3]; + int64_t N = ne_input[3]; + int64_t OH = calc_conv_output_size(H, KH, stride0, padding0, dilation0); + int64_t OW = calc_conv_output_size(W, KW, stride0, padding0, dilation0); + + int64_t K = Cout; + int64_t CRS = Cin * KH * KW; + int64_t NPQ = N * OH * OW; + + return K * NPQ * (2 * CRS - 1); + } + + test_conv_2d(std::array ne_input = { 64, 64, 16, 1 }, + std::array ne_kernel = { 3, 3, 1, 16 }, ggml_type type_kernel = GGML_TYPE_F32, int stride0 = 1, + int stride1 = 1, int padding0 = 0, int padding1 = 0, int dilation0 = 1, int dilation1 = 1, bool cwhn = false) : + ne_input(ne_input), + ne_kernel(ne_kernel), + type_kernel(type_kernel), + stride0(stride0), + stride1(stride1), + padding0(padding0), + padding1(padding1), + dilation0(dilation0), + dilation1(dilation1), + cwhn(cwhn) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * input = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_input.data()); + ggml_set_name(input, "input"); + + ggml_tensor * kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel.data()); + ggml_set_name(kernel, "kernel"); + + if (cwhn) { + // change memory layout to channel-most-contiguous (CWHN), + // then permute it back so NE matches the original input + input = ggml_cont(ctx, ggml_permute(ctx, input, 1, 2, 0, 3)); + input = ggml_permute(ctx, input, 2, 0, 1, 3); + kernel = ggml_cont(ctx, ggml_permute(ctx, kernel, 2, 3, 1, 0)); + kernel = ggml_permute(ctx, kernel, 3, 2, 0, 1); + } + + ggml_tensor * out = + ggml_conv_2d_direct(ctx, kernel, input, stride0, stride1, padding0, padding1, dilation0, dilation1); + ggml_set_name(out, "out"); + return out; + } +}; + +// GGML_OP_CONV_2D_DW +struct test_conv_2d_dw : public test_case { + const std::array ne_input; + const std::array ne_kernel; + const ggml_type type_kernel; + const int stride; + const int padding; + const int dilation; + const bool cwhn; + + std::string vars() override { + return VARS_TO_STR7(ne_input, ne_kernel, type_kernel, stride, padding, dilation, cwhn); + } + + test_conv_2d_dw( + std::array ne_input = {64, 64, 16, 1}, + std::array ne_kernel = {3, 3, 1, 16}, + ggml_type type_kernel = GGML_TYPE_F32, + int stride = 1, int padding = 0, int dilation = 1, bool cwhn = false) + : ne_input(ne_input), ne_kernel(ne_kernel), type_kernel(type_kernel), stride(stride), padding(padding), dilation(dilation), cwhn(cwhn) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * input = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_input.data()); + ggml_set_name(input, "input"); + + ggml_tensor * kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel.data()); + ggml_set_name(kernel, "kernel"); + + if (cwhn) { + // change memory layout to channel-most-contiguous (CWHN), + // then permute it back so NE matches the original input + input = ggml_cont(ctx, ggml_permute(ctx, input, 1, 2, 0, 3)); + input = ggml_permute(ctx, input, 2, 0, 1, 3); + kernel = ggml_cont(ctx, ggml_permute(ctx, kernel, 2, 3, 1, 0)); + kernel = ggml_permute(ctx, kernel, 3, 2, 0, 1); + } + + ggml_tensor * out = ggml_conv_2d_dw_direct( + ctx, kernel, input, + stride, stride, padding, padding, dilation, dilation); + ggml_set_name(out, "out"); + return out; + } +}; + +// GGML_OP_CONV_3D +struct test_conv_3d : public test_case { + // Logical 5D dimensions + const int64_t N, IC, ID, IH, IW; + const int64_t OC, KD, KH, KW; + // Conv params + const int s0, s1, s2; + const int p0, p1, p2; + const int d0, d1, d2; + // Types + const ggml_type type_kernel; + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "CONV_3D"; + } + + std::string vars() override { + return VARS_TO_STR11(N, IC, ID, IH, IW, OC, KD, KH, KW, s0, s1) + "," + + VARS_TO_STR8(s2, p0, p1, p2, d0, d1, d2, type_kernel); + } + + double max_nmse_err() override { + return 5e-4; + } + + uint64_t op_flops(ggml_tensor * t) override { + GGML_UNUSED(t); + auto calc_conv_output_size = [](int64_t ins, int64_t ks, int s, int p, int d) -> int64_t { + return (ins + 2 * p - d * (ks - 1) - 1) / s + 1; + }; + const int64_t OD = calc_conv_output_size(ID, KD, s2, p2, d2); + const int64_t OH = calc_conv_output_size(IH, KH, s1, p1, d1); + const int64_t OW = calc_conv_output_size(IW, KW, s0, p0, d0); + + return (uint64_t)N * OC * OD * OH * OW * (2 * IC * KD * KH * KW - 1); + } + + test_conv_3d( + int64_t N, int64_t IC, int64_t ID, int64_t IH, int64_t IW, + int64_t OC, int64_t KD, int64_t KH, int64_t KW, + int s0, int s1, int s2, + int p0, int p1, int p2, + int d0, int d1, int d2, + ggml_type type_kernel + ) : N(N), IC(IC), ID(ID), IH(IH), IW(IW), + OC(OC), KD(KD), KH(KH), KW(KW), + s0(s0), s1(s1), s2(s2), + p0(p0), p1(p1), p2(p2), + d0(d0), d1(d1), d2(d2), + type_kernel(type_kernel) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + // GGML input tensor is packed as [W, H, D, C*N] + const int64_t ne_input[] = {IW, IH, ID, IC * N}; + ggml_tensor * input = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_input); + ggml_set_name(input, "input"); + + // GGML kernel tensor is packed as [KW, KH, KD, IC*OC] + const int64_t ne_kernel[] = {KW, KH, KD, IC * OC}; + ggml_tensor * kernel = ggml_new_tensor(ctx, type_kernel, 4, ne_kernel); + ggml_set_name(kernel, "kernel"); + + ggml_tensor * out = ggml_conv_3d_direct(ctx, kernel, input, s0, s1, s2, p0, p1, p2, d0, d1, d2, (int)IC, (int)N, (int)OC); + ggml_set_name(out, "out"); + return out; + } +}; + +// GGML_OP_CONCAT +struct test_concat : public test_case { + const ggml_type type; + const std::array ne_a; + const int64_t ne_b_d; + const int dim; + const int v; // view (1 << 0: non-cont a, 1 << 1: non-cont b) + + std::string vars() override { + return VARS_TO_STR5(type, ne_a, ne_b_d, dim, v); + } + + test_concat(ggml_type type = GGML_TYPE_F32, + std::array ne_a = {10, 5, 5, 5}, + int64_t ne_b_d = 5, + int dim = 2, int v = 0) + : type(type), ne_a(ne_a), ne_b_d(ne_b_d), dim(dim), v(v) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + auto ne_b = ne_a; + ne_b[dim] = ne_b_d; + ggml_tensor * a; + if (v & 1) { + auto ne = ne_a; ne[0] *= 2; ne[1] *= 4; ne[2] *= 3; + a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + a = ggml_view_4d(ctx, a, ne_a[0], ne_a[1], ne_a[2], ne_a[3], a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(a, "view_of_a"); + } else { + a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_name(a, "a"); + } + ggml_tensor * b; + if (v & 2) { + auto ne = ne_b; ne[0] *= 3; ne[1] *= 2; ne[2] *= 4; + b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(b, "b"); + + b = ggml_view_4d(ctx, b, ne_b[0], ne_b[1], ne_b[2], ne_b[3], b->nb[1], b->nb[2], b->nb[3], 0); + ggml_set_name(b, "view_of_b"); + } else { + b = ggml_new_tensor(ctx, type, 4, ne_b.data()); + ggml_set_name(b, "b"); + } + + ggml_tensor * out = ggml_concat(ctx, a, b, dim); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_ARGSORT +struct test_argsort : public test_case { + const ggml_type type; + const std::array ne; + ggml_sort_order order; + + std::string vars() override { + return VARS_TO_STR3(type, ne, order); + } + + test_argsort(ggml_type type = GGML_TYPE_F32, + std::array ne = {16, 10, 10, 10}, + ggml_sort_order order = GGML_SORT_ORDER_ASC) + : type(type), ne(ne), order(order) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_argsort(ctx, a, order); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + std::random_device rd; + std::default_random_engine rng(rd()); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_I32) { + // indices + std::vector data(ggml_nelements(t)); + for (int i = 0; i < ggml_nelements(t); i++) { + data[i] = rand(); + } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), 0, ne[0]*ne[1]*ne[2]*ne[3] * sizeof(int)); + } else if (t->type == GGML_TYPE_F32) { + // initialize with unique values to avoid ties + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector data(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + data[i] = i; + } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(float)); + } + } else { + GGML_ABORT("fatal error"); + } + } + } +}; + +// GGML_OP_TOP_K +struct test_top_k : public test_case { + const ggml_type type; + const std::array ne; + const int k; + const bool ties; + ggml_tensor * input {}; + + std::string vars() override { + return VARS_TO_STR4(type, ne, k, ties); + } + + test_top_k(ggml_type type = GGML_TYPE_F32, + std::array ne = {16, 10, 10, 10}, + int k = 4, bool ties = false) + : type(type), ne(ne), k(k), ties(ties) {} + + double max_err() override { + return 0.0; + } + + // When there are ties, only validate the final result. + // The logic in err can't handle the sentinel tensors. + bool run_whole_graph() override { return ties; } + + double err(const float * a, const float * b, size_t n) override { + // When there are no ties, we expect the exact same set of indices, + // but possibly in a different order. When there are ties, the indices + // can be different but the input values they correspond to should be + // the same. The logic for ties could work for non-ties, but only for + // the output tensor, not for the sentinel tensors. + if (ties) { + std::vector src(ggml_nelements(input)); + + ggml_backend_tensor_get(input, src.data(), 0, ggml_nelements(input) * ggml_type_size(type)); + + double diff = 0.0f; + + GGML_ASSERT(n == (size_t)(ggml_nrows(input) * k)); + int64_t cols = input->ne[0]; + std::vector ia(k); + std::vector ib(k); + std::vector asrc(k); + std::vector bsrc(k); + for (int64_t r = 0; r < ggml_nrows(input); r++) { + // Convert indices for the row back to integer + for (int64_t c = 0; c < k; c++) { + ia[c] = (int32_t)a[r * k + c]; + ib[c] = (int32_t)b[r * k + c]; + } + // The src values for each row should match. + for (int64_t c = 0; c < k; c++) { + asrc[c] = src[r * cols + ia[c]]; + bsrc[c] = src[r * cols + ib[c]]; + } + diff += jdst(asrc.data(), bsrc.data(), k); + // There should be no duplicate indices + std::sort(ia.begin(), ia.end()); + std::sort(ib.begin(), ib.end()); + if (std::adjacent_find(ia.begin(), ia.end()) != ia.end()) { + diff += 1; + } + if (std::adjacent_find(ib.begin(), ib.end()) != ib.end()) { + diff += 1; + } + } + return diff; + } else { + std::vector ia(n); + std::vector ib(n); + + double diff = 0.0f; + + for (size_t i = 0; i < n; i++) { + ia[i] = (int32_t) a[i]; + ib[i] = (int32_t) b[i]; + + // penalize the result if the data is not integer valued + diff += std::fabs(a[i] - ia[i]); + diff += std::fabs(b[i] - ib[i]); + } + + return diff + jdst(ia.data(), ib.data(), n); + } + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + // Save 'a' for err() + input = a; + + ggml_tensor * out = ggml_top_k(ctx, a, k); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + std::random_device rd; + std::default_random_engine rng(rd()); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + int tie_denom = std::max(1, std::min(10, k / 2)); + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector data(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + if (ties) { + // integer division to introduce duplicates + data[i] = i / tie_denom; + } else { + data[i] = i; + } + } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(float)); + } + } + } +}; + +enum MoeGatingFunc { + GATING_FUNC_SOFTMAX, + GATING_FUNC_SIGMOID, + GATING_FUNC_SOFTMAX_WEIGHT, +}; + +struct test_topk_moe : public test_case { + const std::array ne; + const int n_expert_used; + const bool with_norm; + const bool bias_probs; + const MoeGatingFunc gating_func; + const float scale_w; + ggml_tensor * weights {}; + ggml_tensor * selected_experts {}; + + test_topk_moe(std::array ne = { 10, 5, 1, 1 }, + int n_expert_used = 1, + bool with_norm = false, + bool bias_probs = false, + MoeGatingFunc gating_func = GATING_FUNC_SOFTMAX, + float scale_w = 0.0f) : + ne(ne), + n_expert_used(n_expert_used), + with_norm(with_norm), + bias_probs(bias_probs), + gating_func(gating_func), + scale_w(scale_w) { + GGML_ASSERT(n_expert_used <= ne[0]); + } + + std::string vars() override { return VARS_TO_STR6(ne, n_expert_used, with_norm, bias_probs, gating_func, scale_w); } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "TOPK_MOE"; + } + + bool run_whole_graph() override { return true; } + + ggml_tensor * build_graph(ggml_context * ctx) override { + const int n_expert = ne[0]; + const int n_tokens = ne[1]; + + ggml_tensor * logits = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data()); + ggml_tensor * probs = + (gating_func == GATING_FUNC_SOFTMAX) ? ggml_soft_max(ctx, logits) : + (gating_func == GATING_FUNC_SIGMOID) ? ggml_sigmoid(ctx, logits) : logits; + ggml_set_name(probs, "probs"); + + ggml_tensor * selection_probs = probs; + if (bias_probs) { + ggml_tensor * exp_probs_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]); + ggml_set_name(exp_probs_b, "exp_probs_b"); + selection_probs = ggml_add(ctx, probs, exp_probs_b); + ggml_set_name(selection_probs, "selection_probs"); + } + + selected_experts = ggml_argsort_top_k(ctx, selection_probs, n_expert_used); // [n_expert_used, n_tokens] + ggml_set_name(selected_experts, "selected_experts"); + + weights = ggml_get_rows(ctx, ggml_reshape_3d(ctx, probs, 1, n_expert, n_tokens), selected_experts); // [1, n_expert_used, n_tokens] + ggml_set_name(weights, "weights"); + + if (gating_func == GATING_FUNC_SOFTMAX_WEIGHT) { + weights = ggml_reshape_2d(ctx, weights, n_expert_used, n_tokens); + weights = ggml_soft_max(ctx, weights); // [n_expert_used, n_tokens] + weights = ggml_reshape_3d(ctx, weights, 1, n_expert_used, n_tokens); + } + + if (with_norm) { + weights = ggml_reshape_2d(ctx, weights, n_expert_used, n_tokens); + ggml_tensor * weights_sum = ggml_sum_rows(ctx, weights); // [1, n_tokens] + ggml_set_name(weights_sum, "weights_sum"); + + weights_sum = ggml_clamp(ctx, weights_sum, 6.103515625e-5, INFINITY); + weights = ggml_div(ctx, weights, weights_sum); // [n_expert_used, n_tokens] + weights = ggml_reshape_3d(ctx, weights, 1, n_expert_used, n_tokens); + } + + if (scale_w) { + weights = ggml_scale(ctx, weights, scale_w); + } + + ggml_set_name(weights, "weights"); + return weights; + } + // Verify two outputs + std::vector fusion_test_nodes() override { return { selected_experts, weights }; } + + // allow output in arbitrary order + double err(const float * a, const float * b, size_t n) override { + std::vector a2(n); + std::vector b2(n); + for (size_t i = 0; i < n; ++i) { + a2[i] = a[i]; + b2[i] = b[i]; + } + std::sort(a2.begin(), a2.end()); + std::sort(b2.begin(), b2.end()); + return nmse(a2.data(), b2.data(), n); + } +}; + +struct test_mul_mat_vec_fusion : public test_case { + const ggml_type type; + const ggml_glu_op glu_op; + const int64_t m; + const int64_t n; + const int64_t k; + const bool use_id; + const int n_mats; + const int n_used; + const bool b; // broadcast b matrix (only for use_id) + const bool with_bias; + const bool with_gate; + const bool with_lane_scale; + std::array batch_dims; + + test_mul_mat_vec_fusion(ggml_type type, ggml_glu_op op, int64_t m, int64_t n, int64_t k, + bool use_id = false, int n_mats = 1, int n_used = 1, bool b = false, bool with_bias = false, bool with_gate = true, + bool with_lane_scale = false, std::array batch_dims = {4, 2}) + : type(type), glu_op(op), m(m), n(n), k(k), use_id(use_id), n_mats(n_mats), n_used(n_used), b(b), with_bias(with_bias), + with_gate(with_gate), with_lane_scale(with_lane_scale), batch_dims(batch_dims) { + if (use_id) { + GGML_ASSERT(n_used <= n_mats); + } + } + + std::string vars() override { + return VARS_TO_STR13(type, glu_op, m, n, k, use_id, n_mats, n_used, b, with_bias, with_gate, with_lane_scale, batch_dims); + } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MUL_MAT_VEC_FUSION"; + } + + bool run_whole_graph() override { return true; } + bool use_weight_context() override { return use_id && with_lane_scale; } + + ggml_tensor * build_gate(ggml_context * ctx, ggml_tensor * ffn_gate, ggml_tensor * ffn_up) { + ggml_tensor * out = nullptr; + if (with_gate) { + if (glu_op == GGML_GLU_OP_SWIGLU_OAI) { + constexpr float alpha = 1.702f; + constexpr float limit = 7.0f; + out = ggml_swiglu_oai(ctx, ffn_gate, ffn_up, alpha, limit); + } else { + out = ggml_glu_split(ctx, ffn_gate, ffn_up, glu_op); + } + } + return out; + } + + ggml_tensor * build_lane_scale_dense(ggml_context * ctx, ggml_tensor * out) { + ggml_tensor * scale = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); + return ggml_mul(ctx, out, scale); + } + + ggml_tensor * build_lane_scale_id(ggml_context * ctx, ggml_context * ctx_weights, ggml_tensor * out, ggml_tensor * ids) { + GGML_ASSERT(ctx_weights); + ggml_tensor * scale = ggml_new_tensor_1d(ctx_weights, GGML_TYPE_F32, n_mats); + ggml_tensor * s = ggml_reshape_3d(ctx, scale, 1, n_mats, 1); + s = ggml_repeat_4d(ctx, s, 1, n_mats, m, 1); + s = ggml_get_rows(ctx, s, ids); + return ggml_mul(ctx, out, s); + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + GGML_ASSERT(!use_weight_context()); + return build_graph(ctx, nullptr); + } + + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + if (!use_id) { + const int channels = batch_dims[0]; + const int samples = batch_dims[1]; + std::array ne = { k, m, channels, samples }; + std::array ne0 = { k, n, channels, samples }; + + ggml_tensor * cur = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data()); + ggml_tensor * gate = with_gate ? ggml_new_tensor(ctx, type, 4, ne0.data()) : nullptr; + ggml_tensor * up = ggml_new_tensor(ctx, type, 4, ne0.data()); + + auto build_lane_up = [&]() { + ggml_tensor * ffn_up = ggml_mul_mat(ctx, up, cur); + if (with_lane_scale) { + ffn_up = build_lane_scale_dense(ctx, ffn_up); + } + if (with_bias) { + std::array bias_ne = { ffn_up->ne[0], 1, channels, samples }; + ggml_tensor * up_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data()); + ffn_up = ggml_add(ctx, ffn_up, up_bias); + } + return ffn_up; + }; + + auto build_lane_gate = [&]() { + ggml_tensor * ffn_gate = ggml_mul_mat(ctx, gate, cur); + if (with_lane_scale) { + ffn_gate = build_lane_scale_dense(ctx, ffn_gate); + } + if (with_bias) { + std::array bias_ne = { ffn_gate->ne[0], 1, channels, samples }; + ggml_tensor * gate_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data()); + ffn_gate = ggml_add(ctx, ffn_gate, gate_bias); + } + return ffn_gate; + }; + + ggml_tensor * ffn_up = build_lane_up(); + ggml_tensor * ffn_gate = with_gate ? build_lane_gate() : nullptr; + + ggml_tensor * out = with_gate ? build_gate(ctx, ffn_gate, ffn_up) : ffn_up; + + std::array bias2_ne = { out->ne[0], 1, channels, samples }; + ggml_tensor * bias2 = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias2_ne.data()); + out = ggml_add(ctx, out, bias2); + + ggml_set_name(out, "out"); + return out; + } else { + ggml_tensor * gates = ggml_new_tensor_3d(ctx, type, k, n, n_mats); + ggml_tensor * ups = ggml_new_tensor_3d(ctx, type, k, n, n_mats); + ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, m); + + if (n_used != n_mats) { + ids = ggml_view_2d(ctx, ids, n_used, m, ids->nb[1], 0); + } + + ggml_tensor * cur = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, this->b ? 1 : n_used, m); + ggml_set_name(cur, "cur"); + + auto build_lane_up = [&]() { + ggml_tensor * ffn_up = ggml_mul_mat_id(ctx, ups, cur, ids); + if (with_lane_scale) { + ffn_up = build_lane_scale_id(ctx, ctx_weights, ffn_up, ids); + } + if (with_bias) { + ggml_tensor * up_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_up->ne[0], n_mats); + ffn_up = ggml_add_id(ctx, ffn_up, up_bias_param, ids); + } + return ffn_up; + }; + + auto build_lane_gate = [&]() { + ggml_tensor * ffn_gate = ggml_mul_mat_id(ctx, gates, cur, ids); + if (with_lane_scale) { + ffn_gate = build_lane_scale_id(ctx, ctx_weights, ffn_gate, ids); + } + if (with_bias) { + ggml_tensor * gate_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_gate->ne[0], n_mats); + ffn_gate = ggml_add_id(ctx, ffn_gate, gate_bias_param, ids); + } + return ffn_gate; + }; + + ggml_tensor * ffn_up = build_lane_up(); + ggml_tensor * ffn_gate = with_gate ? build_lane_gate() : nullptr; + + ggml_tensor * out = with_gate ? build_gate(ctx, ffn_gate, ffn_up) : ffn_up; + + std::array scale_ne { 1, out->ne[1], out->ne[2], out->ne[3] }; + ggml_tensor * scale = ggml_new_tensor(ctx, out->type, 4, scale_ne.data()); + out = ggml_mul(ctx, out, scale); + + ggml_set_name(out, "out"); + return out; + } + } + + void initialize_tensors(ggml_context * ctx) override { + if (!use_id) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t); + } + } else { + init_mul_mat_id_tensors(ctx, n_mats); + } + } + + double max_nmse_err() override { + return 5e-3; + } +}; + +// GGML_OP_SUM +struct test_sum : public test_case { + const ggml_type type; + const std::array ne; + const std::array permute; + bool _use_permute; + + std::string vars() override { + std::string v = VARS_TO_STR2(type, ne); + if (_use_permute) v += "," + VAR_TO_STR(permute); + return v; + } + + test_sum(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 5, 4, 3}, + std::array permute = {0, 0, 0, 0}) + : type(type), ne(ne), permute(permute), + _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + if (_use_permute) { + a = ggml_permute(ctx, a, permute[0], permute[1], permute[2], permute[3]); + ggml_set_name(a, "a_permuted"); + } + + ggml_tensor * out = ggml_sum(ctx, a); + ggml_set_name(out, "out"); + + return out; + } + + float grad_eps() override { + return 0.1f * sqrtf(ne[0]*ne[1]*ne[2]*ne[3]); + } + + // Don't center the distribution around zero. Helps to avoid catastrophic cancellation. + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -0.9f, 1.1f); + } + } +}; + +// GGML_OP_SUM_ROWS +struct test_sum_rows : public test_case { + const ggml_type type; + const std::array ne; + const bool permute; + const bool slice; + + std::string vars() override { + return VARS_TO_STR4(type, ne, permute, slice); + } + + test_sum_rows(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 5, 4, 3}, + bool permute = false, bool slice = false) + : type(type), ne(ne), permute(permute), slice(slice) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + if (slice) { + a = ggml_view_4d(ctx, a, + ne[0], ne[1], ne[2] / 2, ne[3] - 1, + a->nb[1], a->nb[2] * 2, a->nb[3], /*offset=*/a->nb[3]); + } + if (permute) { + a = ggml_permute(ctx, a, 0, 2, 3, 1); + } + + ggml_tensor * out = ggml_sum_rows(ctx, a); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_MEAN +struct test_mean : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_mean(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 5, 4, 3}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_mean(ctx, a); + ggml_set_name(out, "out"); + + return out; + } + + float grad_eps() override { + return 0.1f * ne[0]*ne[1]*ne[2]*ne[3]; + } + + // Don't center the distribution around zero. Helps to avoid catastrophic cancellation. + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -0.9f, 1.1f); + } + } +}; + +// GGML_OP_UPSCALE +struct test_upscale : public test_case { + const ggml_type type; + const std::array ne; + const int32_t scale_factor; + const bool transpose; + const ggml_scale_mode mode; + + std::string vars() override { + return VARS_TO_STR5(type, ne, scale_factor, mode, transpose); + } + + test_upscale(ggml_type type = GGML_TYPE_F32, + std::array ne = {512, 512, 3, 1}, + int32_t scale_factor = 2, ggml_scale_mode mode = GGML_SCALE_MODE_NEAREST, bool transpose = false) + : type(type), ne(ne), scale_factor(scale_factor), transpose(transpose), mode(mode) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + if (transpose) { + a = ggml_transpose(ctx, a); + ggml_set_name(a, "a_transposed"); + } + + ggml_tensor * out = ggml_upscale(ctx, a, scale_factor, mode); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_UPSCALE (via ggml_interpolate) +struct test_interpolate : public test_case { + const ggml_type type; + const std::array ne; + const std::array ne_tgt; + const ggml_scale_mode mode = GGML_SCALE_MODE_NEAREST; + + std::string vars() override { + return VARS_TO_STR4(type, ne, ne_tgt, mode); + } + + test_interpolate(ggml_type type = GGML_TYPE_F32, + std::array ne = {2, 5, 7, 11}, + std::array ne_tgt = {5, 7, 11, 13}, + ggml_scale_mode mode = GGML_SCALE_MODE_NEAREST) + : type(type), ne(ne), ne_tgt(ne_tgt), mode(mode) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_interpolate(ctx, a, ne_tgt[0], ne_tgt[1],ne_tgt[2], ne_tgt[3], mode); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_GROUP_NORM +struct test_group_norm : public test_case { + const ggml_type type; + const std::array ne; + const int32_t num_groups; + const float eps; + + std::string vars() override { + return VARS_TO_STR4(type, ne, num_groups, eps); + } + + test_group_norm(ggml_type type = GGML_TYPE_F32, + std::array ne = {64, 64, 320, 1}, + int32_t num_groups = 32, + float eps = 1e-6f) + : type(type), ne(ne), num_groups(num_groups), eps(eps) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_group_norm(ctx, a, num_groups, eps); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_GROUP_NORM + GGML_OP_MUL + GGML_OP_ADD +struct test_group_norm_mul_add : public test_case { + const ggml_type type; + const std::array ne; + int num_groups; + float eps; + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "GROUP_NORM_MUL_ADD"; + } + + bool run_whole_graph() override { return true; } + + std::string vars() override { + return VARS_TO_STR4(type, ne, num_groups, eps); + } + + test_group_norm_mul_add(ggml_type type = GGML_TYPE_F32, + std::array ne = {128, 1, 1, 1}, + int num_groups = 4, + float eps = 1e-5f) + : type(type), ne(ne), num_groups(num_groups), eps(eps) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_tensor * w = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(a); ggml_set_param(w); ggml_set_param(b); + ggml_set_name(a, "a"); ggml_set_name(w, "w"); ggml_set_name(b, "b"); + ggml_tensor * n = ggml_group_norm(ctx, a, num_groups, eps); + ggml_tensor * m = ggml_mul(ctx, n, w); + ggml_tensor * out = ggml_add(ctx, m, b); + ggml_set_name(out, "out"); + return out; + } +}; + +// GGML_OP_L2_NORM +struct test_l2_norm : public test_case { + const ggml_type type; + const std::array ne; + const float eps; + bool v; + bool noncontig_rows; + + std::string vars() override { + return VARS_TO_STR5(type, ne, eps, v, noncontig_rows); + } + + test_l2_norm(ggml_type type = GGML_TYPE_F32, + std::array ne = {64, 64, 320, 1}, + float eps = 1e-12f, + bool v = false, + bool noncontig_rows = false) + : type(type), ne(ne), eps(eps), v(v), noncontig_rows(noncontig_rows) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const std::array ne_a = noncontig_rows ? + std::array{ ne[1], ne[0], ne[2], ne[3] } : ne; + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_name(a, "a"); + + if (noncontig_rows) { + a = ggml_permute(ctx, a, 1, 0, 2, 3); + ggml_set_name(a, "permuted a"); + } + if (v) { + a = ggml_view_4d(ctx, a, a->ne[0]/2, a->ne[1]/2, a->ne[2]/2, a->ne[3]/2, a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(a, "view of a"); + } + + ggml_tensor * out = ggml_l2_norm(ctx, a, eps); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_ACC +struct test_acc : public test_case { + const ggml_type type; + const std::array ne_a; + const std::array ne_b; + const int64_t stride_dim; + + std::string vars() override { + return VARS_TO_STR4(type, ne_a, ne_b, stride_dim); + } + + test_acc(ggml_type type = GGML_TYPE_F32, + std::array ne_a = {256, 17, 2, 3}, + std::array ne_b = {256, 16, 2, 3}, + uint64_t stride_dim = -1) + : type(type), ne_a(ne_a), ne_b(ne_b), stride_dim(stride_dim) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * b; + if (stride_dim == 1 || stride_dim == 2 || stride_dim == 3) { + // Create a larger tensor and take a view at a non-zero offset. + // This tests that the backend correctly handles b's data offset + std::array ne_b_pad = {ne_b[0], ne_b[1], ne_b[2], ne_b[3]}; + ne_b_pad[stride_dim] += 1; + ggml_tensor * b_pad = ggml_new_tensor(ctx, type, 4, ne_b_pad.data()); + ggml_set_param(b_pad); + ggml_set_name(b_pad, "b_pad"); + // View that skips the first row, so b has a non-zero byte offset + b = ggml_view_4d(ctx, b_pad, + ne_b[0], ne_b[1], ne_b[2], ne_b[3], + b_pad->nb[1], b_pad->nb[2], b_pad->nb[3], + b_pad->nb[1]); + } else { + b = ggml_new_tensor(ctx, type, 4, ne_b.data()); + ggml_set_param(b); + } + ggml_set_name(b, "b"); + + // When ne_b[0] < ne_a[0], a->nb[1] != b->nb[1], so the stride + // parameters to ggml_acc don't match b's natural stride. + ggml_tensor * out = ggml_acc(ctx, a, b, a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_PAD +struct test_pad : public test_case { + const ggml_type type; + const std::array ne_a; + const int pad_0; + const int pad_1; + const bool circular; + + std::string vars() override { + return VARS_TO_STR5(type, ne_a, pad_0, pad_1, circular); + } + + test_pad(ggml_type type = GGML_TYPE_F32, + std::array ne_a = {512, 512, 1, 1}, + int pad_0 = 1, int pad_1 = 1, bool circular = false) + : type(type), ne_a(ne_a), pad_0(pad_0), pad_1(pad_1), circular(circular) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_name(a, "a"); + + ggml_tensor * out = circular + ? ggml_pad_circular(ctx, a, pad_0, pad_1, 0, 0) + : ggml_pad(ctx, a, pad_0, pad_1, 0, 0); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_PAD (with extension) +struct test_pad_ext : public test_case { + const ggml_type type; + const std::array ne_a; + const int lp0; + const int rp0; + const int lp1; + const int rp1; + const int lp2; + const int rp2; + const int lp3; + const int rp3; + const int tfrm; // 0 - none, 1 - non-cont, 2 - perm + const bool circular; + + std::string vars() override { + return VARS_TO_STR12(type, ne_a, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3, tfrm, circular); + } + + test_pad_ext(ggml_type type = GGML_TYPE_F32, + std::array ne_a = {512, 512, 3, 1}, + int lp0 = 1, int rp0 = 1, int lp1 = 1, int rp1 = 1, + int lp2 = 1, int rp2 = 1, int lp3 = 1, int rp3 = 1, + int tfrm = 0, bool circular = false) + : type(type), ne_a(ne_a), lp0(lp0), rp0(rp0), lp1(lp1), rp1(rp1), lp2(lp2), rp2(rp2), lp3(lp3), rp3(rp3), + tfrm(tfrm), circular(circular) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_name(a, "a"); + + if (tfrm == 1) { + a = ggml_view_4d(ctx, a, (a->ne[0] + 1) / 2, (a->ne[1] + 1) / 2, (a->ne[2] + 1) / 2, (a->ne[3] + 1) / 2, a->nb[1], a->nb[2], a->nb[3], 0); + ggml_set_name(a, "view of a"); + } else if (tfrm == 2) { + a = ggml_permute(ctx, a, 2, 1, 0, 3); + ggml_set_name(a, "permuted a"); + } + + ggml_tensor * out = circular + ? ggml_pad_ext_circular(ctx, a, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3) + : ggml_pad_ext (ctx, a, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_PAD_REFLECT_1D +struct test_pad_reflect_1d : public test_case { + const ggml_type type; + const std::array ne_a; + const int pad_0; + const int pad_1; + + std::string vars() override { + return VARS_TO_STR4(type, ne_a, pad_0, pad_1); + } + + test_pad_reflect_1d(ggml_type type = GGML_TYPE_F32, + std::array ne_a = {512, 34, 2, 1}, + int pad_0 = 10, int pad_1 = 9) + : type(type), ne_a(ne_a), pad_0(pad_0), pad_1(pad_1) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 2, ne_a.data()); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_pad_reflect_1d(ctx, a, pad_0, pad_1); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_ROLL +struct test_roll : public test_case { + const int shift0; + const int shift1; + const int shift3; + const int shift4; + + std::string vars() override { + return VARS_TO_STR4(shift0, shift1, shift3, shift4); + } + + test_roll(int shift0 = 3, int shift1 = -2, int shift3 = 1, int shift4 = -1) + : shift0(shift0), shift1(shift1), shift3(shift3), shift4(shift4) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + int64_t ne[4] = {10, 5, 4, 3}; + ggml_tensor * a = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_roll(ctx, a, shift0, shift1, shift3, shift4); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_ARANGE +struct test_arange : public test_case { + const ggml_type type; + const float start; + const float stop; + const float step; + + std::string vars() override { + return VARS_TO_STR4(type, start, stop, step); + } + + test_arange(ggml_type type = GGML_TYPE_F32, + float start = 0.f, float stop = 10.f, float step = 1.f) + : type(type), start(start), stop(stop), step(step) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * out = ggml_arange(ctx, start, stop, step); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_TIMESTEP_EMBEDDING +struct test_timestep_embedding : public test_case { + const ggml_type type; + const std::array ne_a; + const int dim; + const int max_period; + + std::string vars() override { + return VARS_TO_STR4(type, ne_a, dim, max_period); + } + + test_timestep_embedding(ggml_type type = GGML_TYPE_F32, + std::array ne_a = {2, 1, 1, 1}, + int dim = 320, int max_period=10000) + : type(type), ne_a(ne_a), dim(dim), max_period(max_period) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_timestep_embedding(ctx, a, dim, max_period); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_LEAKY_RELU +struct test_leaky_relu : public test_case { + const ggml_type type; + const std::array ne_a; + const float negative_slope; + + std::string vars() override { + return VARS_TO_STR3(type, ne_a, negative_slope); + } + + test_leaky_relu(ggml_type type = GGML_TYPE_F32, + std::array ne_a = {10, 5, 4, 3}, + float negative_slope = 0.1f) + : type(type), ne_a(ne_a), negative_slope(negative_slope) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data()); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_leaky_relu(ctx, a, negative_slope, true); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_FLASH_ATTN_EXT +struct test_flash_attn_ext : public test_case { + const int64_t hsk; // K head size + const int64_t hsv; // V head size + const int64_t nh; // num heads + const std::array nr23; // repeat in dim 2 and 3, tests for grouped-query attention + const int64_t kv; // kv size + const int64_t nb; // batch size + + const bool mask; // use mask + const bool sinks; // use sinks + + const float max_bias; // ALiBi + const float logit_softcap; // Gemma 2 + + const ggml_prec prec; + const ggml_type type_K; + const ggml_type type_V; + std::array permute; + + std::string vars() override { + return VARS_TO_STR14(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute); + } + + double max_nmse_err() override { + return 5e-4; + } + + uint64_t op_flops(ggml_tensor * t) override { + GGML_UNUSED(t); + // Just counting matmul costs: + // Q*K^T is nb x hsk x kv, P*V is nb x kv x hsv, per head + return (2 * nh*nr23[0] * nb * (hsk + hsv) * kv)*nr23[1]; + } + + test_flash_attn_ext(int64_t hsk = 128, int64_t hsv = 128, int64_t nh = 32, std::array nr23 = {1, 1}, int64_t kv = 96, int64_t nb = 8, + bool mask = true, bool sinks = false, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32, + ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16, std::array permute = {0, 1, 2, 3}) + : hsk(hsk), hsv(hsv), nh(nh), nr23(nr23), kv(kv), nb(nb), mask(mask), sinks(sinks), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec), + type_K(type_K), type_V(type_V), permute(permute) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t hsk_padded = GGML_PAD(hsk, ggml_blck_size(type_K)); + const int64_t hsv_padded = GGML_PAD(hsv, ggml_blck_size(type_V)); + + auto const &create_permuted = [&](ggml_type type, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, bool is_view) -> ggml_tensor * { + int64_t ne[4] = {ne0, ne1, ne2, ne3}; + int64_t ne_perm[4]; + for (int i = 0; i < 4; ++i) { + ne_perm[permute[i]] = ne[i]; + } + ggml_tensor * t; + if (is_view) { + ggml_tensor * t0 = ggml_new_tensor_4d(ctx, type, ne_perm[0], 2*ne_perm[1], ne_perm[2], ne_perm[3]); + t = ggml_view_4d(ctx, t0, ne_perm[0], ne_perm[1], ne_perm[2], ne_perm[3], t0->nb[1], t0->nb[2], t0->nb[3], 0); + } else { + t = ggml_new_tensor_4d(ctx, type, ne_perm[0], ne_perm[1], ne_perm[2], ne_perm[3]); + } + if (permute != std::array{0, 1, 2, 3}) { + t = ggml_permute(ctx, t, permute[0], permute[1], permute[2], permute[3]); + } + return t; + }; + + ggml_tensor * q = create_permuted(GGML_TYPE_F32, hsk_padded, nb, nh*nr23[0], nr23[1], false); + ggml_set_name(q, "q"); + + ggml_tensor * k = create_permuted(type_K, hsk_padded, kv, nh, nr23[1], true); // the K tensor is usually a view of the K cache + ggml_set_name(k, "k"); + + ggml_tensor * v = nullptr; + if (type_K == type_V && hsk_padded == 576 && hsv_padded == 512) { + // TODO: this branch should become a separate test case parameter instead of hardcoding this for these head shapes + + // in this branch, the V cache is sub-view of the K cache. this is used by some MLA-based models + // for more info: + // - https://github.com/ggml-org/llama.cpp/pull/13435 + // - https://github.com/ggml-org/llama.cpp/pull/18953#issuecomment-3774948392 + // - https://github.com/ggml-org/llama.cpp/pull/18986 + v = ggml_view_4d(ctx, k, hsv_padded, kv, nh, nr23[1], k->nb[1], k->nb[2], k->nb[3], 0); + } else { + v = create_permuted(type_V, hsv_padded, kv, nh, nr23[1], true); // the V tensor is usually a view of the V cache + } + ggml_set_name(v, "v"); + + ggml_tensor * m = nullptr; + if (mask) { + m = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, kv, nb, 1, nr23[1]); + ggml_set_name(m, "m"); + } + + ggml_tensor * s = nullptr; + if (sinks) { + s = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, q->ne[2]); + ggml_set_name(s, "s"); + } + + ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf(hsk), max_bias, logit_softcap); + ggml_flash_attn_ext_add_sinks(out, s); + ggml_flash_attn_ext_set_prec (out, prec); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "s") == 0) { + // make the sink values more noticeable in order to trigger a test failure when the implementation is wrong + init_tensor_uniform(t, -10.0f, 10.0f); + } else if (strcmp(t->name, "m") == 0) { + init_tensor_kq_mask(t); + } else { + init_tensor_uniform(t); + } + } + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_CROSS_ENTROPY_LOSS +struct test_cross_entropy_loss : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_cross_entropy_loss(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 5, 4, 3}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * logits = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_param(logits); + ggml_set_name(logits, "logits"); + + ggml_tensor * labels = ggml_new_tensor(ctx, type, 4, ne.data()); + // The labels are assumed to be constant -> no gradients. + ggml_set_name(labels, "labels"); + + // Ensure labels add up to 1: + labels = ggml_soft_max(ctx, labels); + ggml_set_name(labels, "labels_normalized"); + + ggml_tensor * out = ggml_cross_entropy_loss(ctx, logits, labels); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + // For larger abs. diffs between logits softmax is more linear, therefore more precise num. gradients. + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -100.0f, 100.0f); + } + } + + float grad_eps() override { + return 1.0f; + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_CROSS_ENTROPY_LOSS_BACK +struct test_cross_entropy_loss_back : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_cross_entropy_loss_back(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 5, 4, 3}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * grad = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); + ggml_set_name(grad, "grad"); + + ggml_tensor * logits = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(logits, "logits"); + + ggml_tensor * labels = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(labels, "labels"); + + // Ensure labels add up to 1: + labels = ggml_soft_max(ctx, labels); + ggml_set_name(labels, "labels_normalized"); + + ggml_tensor * out = ggml_cross_entropy_loss_back(ctx, grad, logits, labels); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_OPT_STEP_ADAMW +struct test_opt_step_adamw : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { + return VARS_TO_STR2(type, ne); + } + + test_opt_step_adamw(ggml_type type = GGML_TYPE_F32, + std::array ne = {10, 5, 4, 3}) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_param(a); // Despite tensor a having gradients the output tensor will not. + ggml_set_name(a, "a"); + + ggml_tensor * grad = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_name(grad, "grad"); + + ggml_tensor * grad_m = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_name(grad_m, "grad_m"); + + ggml_tensor * grad_v = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_name(grad_v, "grad_v"); + + ggml_tensor * adamw_params = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 7); + ggml_set_name(adamw_params, "adamw_params"); + + ggml_tensor * out = ggml_opt_step_adamw(ctx, a, grad, grad_m, grad_v, adamw_params); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, 0.0f, 1.0f); // grad_v and adamw_params need non-negative values. + } + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_OPT_STEP_SGD +struct test_opt_step_sgd : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { return VARS_TO_STR2(type, ne); } + + test_opt_step_sgd(ggml_type type = GGML_TYPE_F32, + std::array ne = { 10, 5, 4, 3 }) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_param(a); // Despite tensor a having gradients the output tensor will not. + ggml_set_name(a, "a"); + + ggml_tensor * grad = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_name(grad, "grad"); + + ggml_tensor * sgd_params = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 2); + ggml_set_name(sgd_params, "sgd_params"); + + ggml_tensor * out = ggml_opt_step_sgd(ctx, a, grad, sgd_params); + + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, 0.0f, 1.0f); // sgd_params need non-negative values. + } + } + + bool grad_precise() override { + return true; + } +}; + +// GGML_OP_CUMSUM +struct test_cumsum : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { return VARS_TO_STR2(type, ne); } + + test_cumsum(ggml_type type = GGML_TYPE_F32, + std::array ne = { 10, 5, 4, 3 }) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_cumsum(ctx, a); + + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -1.0f, 1.0f); + } + } +}; + +// GGML_OP_XIELU +struct test_xielu : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { return VARS_TO_STR2(type, ne); } + + test_xielu(ggml_type type = GGML_TYPE_F32, + std::array ne = { 10, 5, 4, 3 }) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_param(a); + ggml_set_name(a, "a"); + + float alpha_n = 4.0f; + float alpha_p = 20.0f; + float beta = 0.5f; + float eps = 0.0000001f; + + ggml_tensor * out = ggml_xielu(ctx, a, alpha_n, alpha_p, beta, eps); + + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -1.0f, 1.0f); + } + } +}; + +// GGML_OP_TRI +struct test_tri : public test_case { + const ggml_type type; + const std::array ne; + const ggml_tri_type tri_type; + + std::string vars() override { return VARS_TO_STR3(type, ne, tri_type); } + + test_tri(ggml_tri_type tri_type, ggml_type type = GGML_TYPE_F32, + std::array ne = { 10, 10, 4, 3 }) + : type(type), ne(ne), tri_type(tri_type) { + GGML_ASSERT(ne[0] == ne[1]); + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_tri(ctx, a, tri_type); + + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -1.0f, 1.0f); + } + } +}; + +// GGML_OP_FILL +struct test_fill : public test_case { + const ggml_type type; + const std::array ne; + float c; + + std::string vars() override { return VARS_TO_STR3(type, ne, c); } + + test_fill(float c, ggml_type type = GGML_TYPE_F32, + std::array ne = { 10, 10, 4, 3 }) + : type(type), ne(ne), c(c) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_fill(ctx, a, c); + + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_SOLVE_TRI +struct test_solve_tri : public test_case { + const ggml_type type; + const std::array ne_lhs; + const std::array ne_rhs; + + std::string vars() override { return VARS_TO_STR3(type, ne_lhs, ne_rhs); } + + uint64_t op_flops(ggml_tensor * t) override { + GGML_UNUSED(t); + int64_t n = ne_lhs[0]; + int64_t k = ne_rhs[0]; + int64_t batch = ne_lhs[2] * ne_lhs[3]; + // n * (n + 1) / 2 non-zero elements of lhs, 2 flops each, for each col of rhs + return n * (n + 1) * k * batch; + } + + test_solve_tri(ggml_type type = GGML_TYPE_F32, + std::array ne_lhs = { 10, 10, 4, 3 }, + std::array ne_rhs = { 3, 10, 4, 3 } + ) + : type(type), ne_lhs(ne_lhs), ne_rhs(ne_rhs) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne_lhs[0], ne_lhs[1], ne_lhs[2], ne_lhs[3]); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * b = ggml_new_tensor_4d(ctx, type, ne_rhs[0], ne_rhs[1], ne_rhs[2], ne_rhs[3]); + ggml_set_param(b); + ggml_set_name(b, "b"); + + ggml_tensor * out = ggml_solve_tri(ctx, a, b, true, true, false); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "a") == 0) { + // note: avoid zeros in the diagonal + init_tensor_tril(t, 0.1, 1.0f); + } else { + init_tensor_uniform(t, -1.0f, 1.0f); + } + } + } +}; + +// GGML_OP_DIAG +struct test_diag : public test_case { + const ggml_type type; + const std::array ne; + + std::string vars() override { return VARS_TO_STR2(type, ne); } + + test_diag(ggml_type type = GGML_TYPE_F32, + std::array ne = { 10, 1, 4, 3 }) + : type(type), ne(ne) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + GGML_ASSERT(ne[1] == 1); + ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + ggml_set_param(a); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_diag(ctx, a); + ggml_set_name(out, "out"); + + return out; + } +}; + +// GGML_OP_LIGHTNING_INDEXER +struct test_lightning_indexer : public test_case { + const int64_t hsk; // indexer K head size + const int64_t nh; // num indexer heads + const int64_t kv; // kv size + const int64_t nb; // batch size + const int64_t ns; // num streams + const int64_t nm; // ne[3] of mask + + const ggml_type type_K; + + std::string vars() override { + return VARS_TO_STR7(hsk, nh, kv, nb, ns, nm, type_K); + } + + double max_nmse_err() override { + return 1e-6; + } + + uint64_t op_flops(ggml_tensor * t) override { + GGML_UNUSED(t); + return ((2 * hsk + 2) * nh + 1) * kv * nb * ns; + } + + test_lightning_indexer(int64_t hsk = 128, int64_t nh = 64, int64_t kv = 256, int64_t nb = 128, int64_t ns = 1, int64_t nm = 1, ggml_type type_K = GGML_TYPE_F16) + : hsk(hsk), nh(nh), kv(kv), nb(nb), ns(ns), nm(nm), type_K(type_K) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, hsk, nh, nb, ns); + ggml_set_param(q); + ggml_set_name(q, "q"); + + ggml_tensor * k = ggml_new_tensor_4d(ctx, type_K, hsk, 1, kv, ns); + ggml_set_param(k); + ggml_set_name(k, "k"); + + ggml_tensor * w = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, nh, nb, 1, ns); + ggml_set_param(w); + ggml_set_name(w, "w"); + + ggml_tensor * m = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, kv, nb, 1, nm); + ggml_set_param(m); + ggml_set_name(m, "m"); + + ggml_tensor * out = ggml_lightning_indexer(ctx, q, k, w, m); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (strcmp(t->name, "m") == 0) { + init_tensor_kq_mask(t); + } else { + init_tensor_uniform(t); + } + } + } +}; + +// Deserializable generic test case +struct input_tensor { + ggml_type type; + std::array ne; + std::array nb; // strides (0 = use default contiguous strides) +}; + +static bool is_non_contiguous(const input_tensor & src) { + if (src.nb[0] == 0) { + return false; + } + const size_t default_nb0 = ggml_type_size(src.type); + const size_t default_nb1 = default_nb0 * (src.ne[0] / ggml_blck_size(src.type)); + const size_t default_nb2 = default_nb1 * src.ne[1]; + const size_t default_nb3 = default_nb2 * src.ne[2]; + return src.nb[0] != default_nb0 || + src.nb[1] != default_nb1 || + src.nb[2] != default_nb2 || + src.nb[3] != default_nb3; +} + +static std::string var_to_str(const std::vector& sources) { + std::ostringstream oss; + bool first = true; + for (const auto& src : sources) { + if (!first) oss << ","; + oss << ggml_type_name(src.type) << "[" << src.ne[0] << "," << src.ne[1] << "," << src.ne[2] << "," << src.ne[3] << "]"; + if (is_non_contiguous(src)) { + oss << "nb[" << src.nb[0] << "," << src.nb[1] << "," << src.nb[2] << "," << src.nb[3] << "]"; + } + first = false; + } + return oss.str(); +} + +static std::string var_to_str(const std::array& params) { + std::ostringstream oss; + oss << "["; + bool first = true; + for (size_t i = 0; i < params.size(); ++i) { + if (params[i] != 0) { + if (!first) oss << ","; + oss << i << ":" << params[i]; + first = false; + } + } + oss << "]"; + return oss.str(); +} + + +struct test_generic_op : public test_case { + const ggml_op op; + const ggml_type type; + const std::array ne; + const std::array op_params; + + const std::vector sources; + const std::string name; + + std::string vars() override { + if (name.empty()) { + return VARS_TO_STR4(type, ne, op_params, sources); + } + + return VARS_TO_STR5(name, type, ne, op_params, sources); + } + + test_generic_op(ggml_op op, ggml_type type, std::array ne, + std::array op_params, + std::vector sources, std::string name = "") + : op(op), type(type), ne(ne), op_params(op_params), sources(sources), name(std::move(name)) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const size_t source_count = std::min(sources.size(), (size_t)GGML_MAX_SRC); + + std::array source_tensors; + for (size_t i = 0; i < source_count; ++i) { + const input_tensor& src = sources[i]; + + if (is_non_contiguous(src)) { + size_t total_size; + const size_t blck_size = ggml_blck_size(src.type); + if (blck_size == 1) { + total_size = ggml_type_size(src.type); + for (int d = 0; d < 4; d++) { + total_size += (src.ne[d] - 1) * src.nb[d]; + } + } else { + total_size = src.ne[0] * src.nb[0] / blck_size; + for (int d = 1; d < 4; d++) { + total_size += (src.ne[d] - 1) * src.nb[d]; + } + } + + // Convert bytes to elements, padded to block size for quantized types + const size_t type_size = ggml_type_size(src.type); + size_t backing_elements = (total_size * blck_size + type_size - 1) / type_size; + backing_elements = ((backing_elements + blck_size - 1) / blck_size) * blck_size; + ggml_tensor * backing = ggml_new_tensor_1d(ctx, src.type, backing_elements); + source_tensors[i] = ggml_view_4d(ctx, backing, + src.ne[0], src.ne[1], src.ne[2], src.ne[3], + src.nb[1], src.nb[2], src.nb[3], 0); + // nb[0] does not get set by view_4d, so set it manually + source_tensors[i]->nb[0] = src.nb[0]; + } else { + source_tensors[i] = ggml_new_tensor_4d(ctx, src.type, src.ne[0], src.ne[1], src.ne[2], src.ne[3]); + } + } + + // Ops with an inplace flag create a view of src[0] as their output. + bool inplace = false; + if (op == GGML_OP_SET || op == GGML_OP_ACC) { + inplace = op_params[4] != 0; + } else if (op == GGML_OP_ADD_REL_POS) { + inplace = op_params[0] != 0; + } + + ggml_tensor * out; + if (inplace && source_count > 0) { + out = ggml_view_tensor(ctx, source_tensors[0]); + } else { + out = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]); + } + out->op = op; + for (size_t i = 0; i < source_count; ++i) { + out->src[i] = source_tensors[i]; + } + + memcpy(out->op_params, op_params.data(), GGML_MAX_OP_PARAMS); + ggml_set_name(out, "out"); + + return out; + } + + double max_nmse_err() override { + switch (op) { + case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: + case GGML_OP_OUT_PROD: + case GGML_OP_CONV_TRANSPOSE_2D: + case GGML_OP_IM2COL: + case GGML_OP_CONV_2D: + case GGML_OP_CONV_3D: + case GGML_OP_SET_ROWS: + case GGML_OP_CPY: + return 5e-4; + case GGML_OP_SOFT_MAX: + return 1e-6; + case GGML_OP_RWKV_WKV7: + return 5e-3; + case GGML_OP_FLASH_ATTN_EXT: + { + // Scale error with kv length to account for accumulating floating point error + const int64_t kv = sources[1].ne[1]; + return 5e-4 * std::max(1.0, kv / 20000.0); + } + default: + return 1e-7; + } + } + + void initialize_tensors(ggml_context * ctx) override { + ggml_tensor * out = ggml_get_tensor(ctx, "out"); + + std::random_device rd; + std::default_random_engine rng(rd()); + + for (size_t i = 0; i < sources.size() && i < GGML_MAX_SRC; i++) { + ggml_tensor * t = out->src[i]; + if (!t) { + break; + } + + // FLASH_ATTN_EXT: src[3] is the KQ mask + if (op == GGML_OP_FLASH_ATTN_EXT && i == 3) { + init_tensor_kq_mask(t); + continue; + } + + if (t->type == GGML_TYPE_I32 || t->type == GGML_TYPE_I64) { + if (op == GGML_OP_GET_ROWS || op == GGML_OP_GET_ROWS_BACK) { + const int64_t num_rows = sources[0].ne[1]; + const int64_t nels = ggml_nelements(t); + std::vector data(nels); + std::uniform_int_distribution dist(0, num_rows - 1); + for (int64_t i = 0; i < nels; i++) { + data[i] = dist(rng); + } + ggml_backend_tensor_set(t, data.data(), 0, nels * sizeof(int32_t)); + } else if (op == GGML_OP_SET_ROWS) { + init_set_rows_row_ids(t, ne[1]); + } else if (op == GGML_OP_ROPE) { + const int mode = op_params[2]; + const int64_t nels = (mode & GGML_ROPE_TYPE_MROPE) ? ne[2] * 4 : ne[2]; + std::vector data(nels); + std::uniform_int_distribution dist(0, ne[2] - 1); + for (int64_t i = 0; i < nels; i++) { + data[i] = dist(rng); + } + ggml_backend_tensor_set(t, data.data(), 0, nels * sizeof(int32_t)); + } else if (op == GGML_OP_MUL_MAT_ID || op == GGML_OP_ADD_ID) { + const int64_t n_expert = (op == GGML_OP_MUL_MAT_ID) ? sources[0].ne[2] : sources[1].ne[1]; + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector data(t->ne[0]); + for (int32_t i = 0; i < t->ne[0]; i++) { + data[i] = i % n_expert; + } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); + } + } else if (op == GGML_OP_SSM_SCAN) { + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector data(t->ne[0]); + for (int32_t i = 0; i < t->ne[0]; i++) { + data[i] = i; + } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); + } + } else { + init_tensor_uniform(t); + } + } else { + init_tensor_uniform(t); + } + } + } +}; + + +enum llm_norm_type { + LLM_NORM, + LLM_NORM_RMS, +}; + +struct llama_hparams { + uint32_t n_vocab; + uint32_t n_embd; + uint32_t n_head; + uint32_t n_head_kv; + static constexpr uint32_t n_layer = 1; + uint32_t n_rot; + uint32_t n_embd_head; // dimension of values (d_v) + uint32_t n_ff; + + float f_norm_eps; + float f_norm_rms_eps; + + // cparams + static constexpr uint32_t n_ctx = 512; // user-specified context size + static constexpr uint32_t n_ctx_orig = n_ctx; + + // batch + int32_t n_tokens; + + // llm_build_context + static constexpr int32_t n_kv = 32; // size of KV cache to consider (n_kv <= n_ctx + static constexpr int32_t kv_head = 1; // index of where we store new KV data in the cache + + uint32_t n_embd_gqa() const { // dimension of key embeddings across all k-v heads + return n_embd_head * n_head_kv; + } +}; + +// LLM base class +struct test_llm : public test_case { + llama_hparams hp; + +protected: + test_llm(llama_hparams hp) + : hp(std::move(hp)) { + } + +public: + struct ggml_tensor * llm_build_norm( + struct ggml_context * ctx, + struct ggml_tensor * cur, + struct ggml_tensor * mw, + struct ggml_tensor * mb, + llm_norm_type type) { + switch (type) { + case LLM_NORM: cur = ggml_norm (ctx, cur, hp.f_norm_eps); break; + case LLM_NORM_RMS: cur = ggml_rms_norm(ctx, cur, hp.f_norm_rms_eps); break; + } + cur = ggml_mul(ctx, cur, mw); + if (mb) { + cur = ggml_add(ctx, cur, mb); + } + return cur; + } + + void llm_build_kv_store( + struct ggml_context * ctx, + struct ggml_tensor * k_l, + struct ggml_tensor * v_l, + struct ggml_tensor * k_cur, + struct ggml_tensor * v_cur) { + // compute the transposed [n_tokens, n_embd] V matrix + struct ggml_tensor * v_cur_t = ggml_transpose(ctx, ggml_reshape_2d(ctx, v_cur, hp.n_embd_gqa(), hp.n_tokens)); + + struct ggml_tensor * k_cache_view = ggml_view_1d(ctx, k_l, hp.n_tokens*hp.n_embd_gqa(), + (ggml_row_size(k_l->type, hp.n_embd_gqa()))*hp.kv_head); + + struct ggml_tensor * v_cache_view = ggml_view_2d(ctx, v_l, hp.n_tokens, hp.n_embd_gqa(), + ( hp.n_ctx)*ggml_element_size(v_l), + (hp.kv_head)*ggml_element_size(v_l)); + + // important: storing RoPE-ed version of K in the KV cache! + ggml_cpy(ctx, k_cur, k_cache_view); + ggml_cpy(ctx, v_cur_t, v_cache_view); + } + + struct ggml_tensor * llm_build_kqv( + struct ggml_context * ctx, + struct ggml_tensor * k_l, + struct ggml_tensor * v_l, + struct ggml_tensor * q_cur, + struct ggml_tensor * kq_mask, + float kq_scale) { + struct ggml_tensor * q = ggml_permute(ctx, q_cur, 0, 2, 1, 3); + + struct ggml_tensor * k = + ggml_view_3d(ctx, k_l, + hp.n_embd_head, hp.n_kv, hp.n_head_kv, + ggml_row_size(k_l->type, hp.n_embd_gqa()), + ggml_row_size(k_l->type, hp.n_embd_head), + 0); + + struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q); + + kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_scale, 0.0f); + + // split cached v into n_head heads + struct ggml_tensor * v = + ggml_view_3d(ctx, v_l, + hp.n_kv, hp.n_embd_head, hp.n_head_kv, + ggml_element_size(v_l)*hp.n_ctx, + ggml_element_size(v_l)*hp.n_ctx*hp.n_embd_head, + 0); + + struct ggml_tensor * kqv = ggml_mul_mat(ctx, v, kq); + + struct ggml_tensor * kqv_merged = ggml_permute(ctx, kqv, 0, 2, 1, 3); + + struct ggml_tensor * cur = ggml_cont_2d(ctx, kqv_merged, hp.n_embd_head*hp.n_head, hp.n_tokens); + + struct ggml_tensor * wo = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_embd); + cur = ggml_mul_mat(ctx, wo, cur); + + return cur; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_I32) { + // pos + std::vector data(hp.n_tokens); + for (int i = 0; i < hp.n_tokens; i++) { + data[i] = rand() % hp.n_ctx; + } + ggml_backend_tensor_set(t, data.data(), 0, hp.n_tokens * sizeof(int)); + } else { + init_tensor_uniform(t); + } + } + } +}; + +// Llama +struct test_llama : public test_llm { + static constexpr float freq_base = 10000.0f; + static constexpr float freq_scale = 1.0f; + static constexpr float ext_factor = 0.0f; + static constexpr float attn_factor = 1.0f; + static constexpr float beta_fast = 32.0f; + static constexpr float beta_slow = 1.0f; + bool fused; + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "LLAMA"; + } + + std::string vars() override { + auto n_tokens = hp.n_tokens; + return VARS_TO_STR1(n_tokens); + } + + double max_nmse_err() override { + return 2e-3; + } + + bool run_whole_graph() override { return fused; } + + test_llama(int n_tokens = 1, bool fused = false) + : test_llm({ + /*n_vocab =*/ 32000, + /*n_embd =*/ 3200, + /*n_head =*/ 32, + /*n_head_kv =*/ 32, + /*n_rot =*/ 100, + /*n_embd_head =*/ 100, + /*n_ff =*/ 8640, + /*f_norm_eps =*/ 0.f, + /*f_norm_rms_eps =*/ 1e-5f, + /*n_tokens =*/ n_tokens, + }) + , fused(fused) + { + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + inpL = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hp.n_embd, hp.n_tokens); + + // inp_pos - contains the positions + struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, hp.n_tokens); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx, GGML_TYPE_F16, hp.n_kv, hp.n_tokens, 1); + + ggml_tensor * k_l = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, 1638400); + ggml_tensor * v_l = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, 1638400); + + for (uint32_t il = 0; il < hp.n_layer; ++il) { + struct ggml_tensor * inpSA = inpL; + + // norm + ggml_tensor * attn_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); + cur = llm_build_norm(ctx, inpL, attn_norm, nullptr, LLM_NORM_RMS); + + // self-attention + { + ggml_tensor * wq = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_embd); + ggml_tensor * wk = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_embd_gqa()); + ggml_tensor * wv = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_embd_gqa()); + + // compute Q and K and RoPE them + struct ggml_tensor * Qcur = ggml_mul_mat(ctx, wq, cur); + struct ggml_tensor * Kcur = ggml_mul_mat(ctx, wk, cur); + struct ggml_tensor * Vcur = ggml_mul_mat(ctx, wv, cur); + + Qcur = ggml_rope_ext( + ctx, ggml_reshape_3d(ctx, Qcur, hp.n_embd_head, hp.n_head, hp.n_tokens), inp_pos, nullptr, + hp.n_rot, 0, hp.n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + + Kcur = ggml_rope_ext( + ctx, ggml_reshape_3d(ctx, Kcur, hp.n_embd_head, hp.n_head_kv, hp.n_tokens), inp_pos, nullptr, + hp.n_rot, 0, hp.n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + + llm_build_kv_store(ctx, k_l, v_l, Kcur, Vcur); + + cur = llm_build_kqv(ctx, k_l, v_l, Qcur, KQ_mask, 1.0f/sqrtf(float(hp.n_embd_head))); + } + + struct ggml_tensor * ffn_inp = ggml_add(ctx, cur, inpSA); + + // feed-forward network + ggml_tensor * ffn_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); + cur = llm_build_norm(ctx, ffn_inp, ffn_norm, nullptr, LLM_NORM_RMS); + + ggml_tensor * ffn_gate = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_ff); + ggml_tensor * ffn_down = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_ff, hp.n_embd); + ggml_tensor * ffn_up = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_ff); + struct ggml_tensor * tmp = ggml_mul_mat(ctx, ffn_up, cur); + cur = ggml_mul_mat(ctx, ffn_gate, cur); + cur = ggml_silu(ctx, cur); + cur = ggml_mul(ctx, cur, tmp); + cur = ggml_mul_mat(ctx, ffn_down, cur); + + cur = ggml_add(ctx, cur, ffn_inp); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + ggml_tensor * output_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); + cur = llm_build_norm(ctx, cur, output_norm, nullptr, LLM_NORM_RMS); + + // lm_head + ggml_tensor * output = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_vocab); + cur = ggml_mul_mat(ctx, output, cur); + + return cur; + } +}; + +// Falcon +struct test_falcon : public test_llm { + static constexpr float freq_base = 10000.0f; + static constexpr float freq_scale = 1.0f; + static constexpr float ext_factor = 0.0f; + static constexpr float attn_factor = 1.0f; + static constexpr float beta_fast = 32.0f; + static constexpr float beta_slow = 1.0f; + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "FALCON"; + } + + std::string vars() override { + auto n_tokens = hp.n_tokens; + return VARS_TO_STR1(n_tokens); + } + + double max_nmse_err() override { + return 2e-3; + } + + test_falcon(int n_tokens = 1) + : test_llm({ + /*n_vocab =*/ 32000, + /*n_embd =*/ 3200, + /*n_head =*/ 50, + /*n_head_kv =*/ 1, + /*n_rot =*/ 64, + /*n_embd_head =*/ 64, + /*n_ff =*/ 8640, + /*f_norm_eps =*/ 1e-5f, + /*f_norm_rms_eps =*/ 0.f, + /*n_tokens =*/ n_tokens, + }) { + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + inpL = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hp.n_embd, hp.n_tokens); + + // inp_pos - contains the positions + struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, hp.n_tokens); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx, GGML_TYPE_F16, hp.n_kv, hp.n_tokens, 1); + + ggml_tensor * k_l = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, 1638400); + ggml_tensor * v_l = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, 1638400); + + for (uint32_t il = 0; il < hp.n_layer; ++il) { + // norm + ggml_tensor * attn_norm_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); + ggml_tensor * attn_norm_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); + ggml_tensor * attn_norm = llm_build_norm(ctx, inpL, attn_norm_w, attn_norm_b, LLM_NORM); + + // self-attention + { + cur = attn_norm; + + ggml_tensor * wqkv = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_embd + 2*hp.n_embd_gqa()); + + cur = ggml_mul_mat(ctx, wqkv, cur); + + struct ggml_tensor * Qcur = ggml_cont(ctx, ggml_view_2d(ctx, cur, hp.n_embd, hp.n_tokens, cur->nb[1], 0*sizeof(float)*(hp.n_embd))); + struct ggml_tensor * Kcur = ggml_cont(ctx, ggml_view_2d(ctx, cur, hp.n_embd_gqa(), hp.n_tokens, cur->nb[1], 1*sizeof(float)*(hp.n_embd))); + struct ggml_tensor * Vcur = ggml_cont(ctx, ggml_view_2d(ctx, cur, hp.n_embd_gqa(), hp.n_tokens, cur->nb[1], 1*sizeof(float)*(hp.n_embd + hp.n_embd_gqa()))); + + Qcur = ggml_reshape_3d(ctx, Qcur, hp.n_embd_head, hp.n_head, hp.n_tokens); + Kcur = ggml_reshape_3d(ctx, Kcur, hp.n_embd_head, hp.n_head_kv, hp.n_tokens); + + // using mode = 2 for neox mode + Qcur = ggml_rope_ext( + ctx, Qcur, inp_pos, nullptr, hp.n_rot, 2, hp.n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ); + + Kcur = ggml_rope_ext( + ctx, Kcur, inp_pos, nullptr, hp.n_rot, 2, hp.n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow + ); + + llm_build_kv_store(ctx, k_l, v_l, Kcur, Vcur); + + cur = llm_build_kqv(ctx, k_l, v_l, Qcur, KQ_mask, 1.0f/sqrtf(float(hp.n_embd_head))); + } + + struct ggml_tensor * ffn_inp = cur; + + // feed forward + { + ggml_tensor * ffn_up = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_embd, hp.n_ff); + ggml_tensor * ffn_down = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, hp.n_ff, hp.n_embd); + cur = attn_norm; + cur = ggml_mul_mat(ctx, ffn_up, cur); + cur = ggml_gelu(ctx, cur); + cur = ggml_mul_mat(ctx, ffn_down, cur); + } + + cur = ggml_add(ctx, cur, ffn_inp); + + cur = ggml_add(ctx, cur, inpL); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + ggml_tensor * output_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); + ggml_tensor * output_norm_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hp.n_embd); + cur = llm_build_norm(ctx, cur, output_norm, output_norm_b, LLM_NORM); + + // lm_head + ggml_tensor * output = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, hp.n_embd, hp.n_vocab); + cur = ggml_mul_mat(ctx, output, cur); + + return cur; + } +}; + + +// ########################################### +// ## Section 3: GGML Op Test Instantiation ## +// ########################################### +static const ggml_type all_types[] = { + GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, + GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, + GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, + GGML_TYPE_Q8_0, + GGML_TYPE_Q1_0, + GGML_TYPE_MXFP4, GGML_TYPE_NVFP4, + GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, + GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, + GGML_TYPE_Q6_K, + // GGML_TYPE_TQ1_0, GGML_TYPE_TQ2_0, // TODO: implement for all backends + GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, + GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, + GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS, +}; + +static const ggml_type base_types[] = { + GGML_TYPE_F32, GGML_TYPE_F16, + GGML_TYPE_Q8_0, // for I8MM tests + GGML_TYPE_Q1_0, + GGML_TYPE_Q4_0, + GGML_TYPE_Q4_1, // for I8MM tests + GGML_TYPE_Q4_K, + GGML_TYPE_MXFP4, GGML_TYPE_NVFP4, // TODO: or "other" + GGML_TYPE_IQ2_XXS +}; + +static const ggml_type other_types[] = { + GGML_TYPE_Q4_1, + GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, + GGML_TYPE_Q8_0, + GGML_TYPE_Q1_0, + GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, + GGML_TYPE_Q5_K, + GGML_TYPE_Q6_K, + // GGML_TYPE_TQ1_0, GGML_TYPE_TQ2_0, // TODO: implement for all backends + GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, + GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, + GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS, + GGML_TYPE_BF16, +}; + +#ifdef _MSC_VER +// Workaround long compile time with msvc +#pragma optimize("", off) +#endif + +// Test cases for evaluation: should try to cover edge cases while using small input sizes to keep the runtime low +static std::vector> make_test_cases_eval() { + std::vector> test_cases; + std::default_random_engine rng(0); + + // unary ops + for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { + for (int v : {0, 1}) { + for (int op = 0; op < GGML_UNARY_OP_COUNT; op++) { + if (op == GGML_UNARY_OP_XIELU) { + continue; // need extra params, separate test + } + test_cases.emplace_back(new test_unary((ggml_unary_op) op, type, { 128, 2, 2, 2 }, v)); + test_cases.emplace_back(new test_unary((ggml_unary_op) op, type, { 5, 7, 11, 13 }, v)); + } + } + } + + // fused relu + sqr (squared ReLU) + for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { + test_cases.emplace_back(new test_relu_sqr(type, { 128, 2, 2, 2 })); + test_cases.emplace_back(new test_relu_sqr(type, { 5, 7, 11, 13 })); + } + + // SNAKE activation fusion: x + sin(a*x)^2 * inv_b + for (ggml_type type : { GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16 }) { + test_cases.emplace_back(new test_snake_fuse(type, { 5, 7, 1, 1})); // primes sub-block + test_cases.emplace_back(new test_snake_fuse(type, { 33, 32, 1, 1})); // boundary + test_cases.emplace_back(new test_snake_fuse(type, {1025, 13, 1, 1})); // large prime, grid-stride + test_cases.emplace_back(new test_snake_fuse(type, { 128, 16, 1, 1})); // power-of-two + test_cases.emplace_back(new test_snake_fuse(type, { 256, 192, 1, 1})); // BigVGAN-ish + // higher-rank shapes: matcher must reject fusion, fallback to naive chain + test_cases.emplace_back(new test_snake_fuse(type, { 64, 32, 2, 1})); // ne[2] > 1 + test_cases.emplace_back(new test_snake_fuse(type, { 64, 32, 1, 2})); // ne[3] > 1 + test_cases.emplace_back(new test_snake_fuse(type, { 64, 32, 2, 3})); // ne[2] > 1 and ne[3] > 1 + } + + // glu ops + for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { + for (int v : {0, 1}) { + for (int op = 0; op < GGML_GLU_OP_COUNT; op++) { + if (op == GGML_GLU_OP_SWIGLU_OAI) { + // SWIGLU_OAI is handled separately + continue; + } + + for (bool swapped : {false, true}) { + test_cases.emplace_back(new test_glu((ggml_glu_op) op, type, { 128, 2, 2, 2 }, v, swapped)); + test_cases.emplace_back(new test_glu((ggml_glu_op) op, type, { 5, 7, 11, 13 }, v, swapped)); + } + + test_cases.emplace_back(new test_glu_split((ggml_glu_op) op, type, { 128, 2, 2, 2 }, v)); + test_cases.emplace_back(new test_glu_split((ggml_glu_op) op, type, { 5, 7, 11, 13 }, v)); + } + } + } + + for (int v : {0, 1}) { + for (float alpha : {.5f, 1.702f}) { + for (float limit : {2.0f, 7.0f}) { + test_cases.emplace_back(new test_swiglu_oai(GGML_TYPE_F32, { 128, 2, 2, 2 }, v, alpha, limit)); + } + } + } + + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_Q4_0}) { + test_cases.emplace_back(new test_get_rows(type, 300*256, 5, 4, 1, 2, false)); + test_cases.emplace_back(new test_get_rows(type, 256, 80000, 70000, 2, 1, false)); + test_cases.emplace_back(new test_get_rows(type, 256, 5, 4, 700, 100, false)); + } + + test_cases.emplace_back(new test_get_rows(GGML_TYPE_F32, 1, 8, 2, 1, 1, false)); + for (ggml_type type : all_types) { + for (int b : {1, 7}) { + for (bool v : {false, true}) { + test_cases.emplace_back(new test_get_rows(type, 256, 5, 4, b, 1, v)); + } + } + } + for (int b : {1, 7}) { + for (bool v : {false, true}) { + test_cases.emplace_back(new test_get_rows(GGML_TYPE_I32, 256, 5, 4, b, 1, v)); + } + } + + test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 8, 2, 1, false)); + test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 70000, 4, 1, false)); // row count > CUDA grid-y limit (65535) + for (ggml_type type : all_types) { + for (bool v : {false, true}) { + test_cases.emplace_back(new test_get_rows_back(type, 256, 5, 4, 1, v)); + } + } + for (bool v : {false, true}) { + test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_I32, 256, 5, 4, 1, v)); + } + + test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); + test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); + test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_Q8_0, GGML_TYPE_I32, { 256, 5, 1, 3 }, { 1, 1, }, 1, false)); + for (ggml_type type : all_types) { + for (int b : {1, 7}) { + for (bool v : {false, true}) { + test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 256, 5, b, 3 }, { 1, 1, }, 1, v)); + test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 256, 11, 1, b }, { 2, 3, }, 7, v)); + + test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 3*ggml_blck_size(type), 3, b, 1 }, { 2, 3, }, 2, v)); + + if (ggml_blck_size(type) == 1) { + test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 31, 3, b, 1 }, { 2, 3, }, 2, v)); + test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 33, 5, 1, b }, { 2, 3, }, 1, v)); + } + } + } + } + test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); + test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); + test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, true)); + test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, true)); + + for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_VISION }) { + for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { + for (int ne2 : {1, 8, 512}) { + test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, { 128, 32, ne2, 1 }, mode)); + test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, { 128, 32, ne2, 3 }, mode)); + } + } + } + + for (ggml_type type_input : {GGML_TYPE_F32}) { + for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) { + for (int k0 : {1, 3}) { + for (int k1 : {1, 3}) { + for (int s0 : {1, 2}) { + for (int s1 : {1, 2}) { + for (int p0 : {0, 1}) { + for (int p1 : {0, 1}) { + test_cases.emplace_back(new test_pool2d(pool_type, type_input, {10, 10, 3, 1}, k0, k1, s0, s1, p0, p1)); + } + } + } + } + } + } + } + } + + for (ggml_type type_input : {GGML_TYPE_F32}) { + for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) { + for (int k0 : {1, 3}) { + for (int s0 : {1, 2}) { + for (int p0 : {0, 1}) { + test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 10, 3, 2, 1 }, k0, s0, p0)); + test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 11, 1, 3, 2 }, k0, s0, p0)); + test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 128, 2, 1, 3 }, k0, s0, p0)); + } + } + } + } + } + +#if 0 + // >4GB im2col destination. Too slow to run by default. + // Test cases taken from Wan2.1 T2V 1.3B. + test_cases.emplace_back(new test_im2col (GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {832, 480, 192, 4}, {3, 3, 192, 96}, 1, 1, 1, 1, 1, 1, true)); + test_cases.emplace_back(new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {834, 482, 6, 96}, {3, 3,3, 9216}, 96, 1, 1, 1, 0, 0, 0, 1, 1, 1, false)); +#endif + + // im2col 1D + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {3000, 384, 1, 1}, {3, 384, 384, 1}, 1, 0, 1, 0, 1, 0, false)); + for (int s0 : {1, 3}) { + for (int p0 : {0, 3}) { + for (int d0 : {1, 3}) { + test_cases.emplace_back(new test_im2col( + GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {20, 2, 2, 1}, {3, 2, 2, 1}, + s0, 0, p0, 0, d0, 0, false)); + } + } + } + + // im2col 2D + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + for (int s0 : {1, 3}) { + for (int s1 : {1, 3}) { + for (int p0 : {0, 3}) { + for (int p1 : {0, 3}) { + for (int d0 : {1, 3}) { + for (int d1 : {1, 3}) { + test_cases.emplace_back(new test_im2col( + GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {20, 20, 2, 2}, {3, 3, 2, 2}, + s0, s1, p0, p1, d0, d1, true)); + } + } + } + } + } + } + + // extra tests for im2col 2D + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 1, 32}, {3, 3, 1, 32}, 1, 1, 1, 1, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 2, 32}, {3, 3, 2, 32}, 1, 1, 1, 1, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 1, 1024}, {3, 3, 1, 1024}, 1, 1, 1, 1, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 2, 1024}, {3, 3, 2, 1024}, 1, 1, 1, 1, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 1, 2048}, {3, 3, 1, 2048}, 1, 1, 1, 1, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 2, 2048}, {3, 3, 2, 2048}, 1, 1, 1, 1, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 1, 2560}, {3, 3, 1, 2560}, 1, 1, 1, 1, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 2, 2560}, {3, 3, 2, 2560}, 1, 1, 1, 1, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {5, 5, 1, 32}, {3, 4, 1, 32}, 1, 1, 0, 0, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {2, 2, 1536, 729}, {2, 2, 1536, 4096}, 1, 1, 0, 0, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {128, 128, 1, 2}, {32, 33, 1, 2}, 1, 1, 1, 1, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {128, 128, 2, 1}, {33, 34, 2, 1}, 1, 1, 1, 1, 1, 1, true)); + + // im2col 3D + test_cases.emplace_back(new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32)); + test_cases.emplace_back(new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32)); + test_cases.emplace_back(new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + for (int s0 : {1, 3}) { + for (int s1 : {1, 3}) { + for (int s2 : {1, 3}) { + for (int p0 : {0, 3}) { + for (int p1 : {0, 3}) { + for (int p2 : {0, 3}) { + for (int d0 : {1, 3}) { + for (int d1 : {1, 3}) { + for (int d2 : {1, 3}) { + for (int IC : {1, 3}) { + for (bool v : {false, true}) { + test_cases.emplace_back(new test_im2col_3d( + GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {20, 20, 10, 3}, {3, 3, 3, 3}, + IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, v)); + } + } + } + } + } + } + } + } + } + } + } + +// Conv_2D test cases +#ifdef DETAILED_TESTS + // Probably we do not have enough time to execute these in the pipeline. + uint32_t iwh_idx = 0; + uint32_t kwh_idx = 1; + uint32_t Cout_idx = 2; + uint32_t Cin_idx = 3; + uint32_t B_idx = 4; + + std::vector> cases = { + //{IWH, KWH, Cout, Cin, B} + // K=CRS=NPQ=4096 conv_2d matmul performance + {19, 4, 4096, 256, 16}, + // K=128, CRS=128, NPQ=4096 + { 19, 4, 128, 8, 16}, + // K=130, CRS=128, NPQ=4096 + { 19, 4, 130, 8, 16}, + // Edge case: K x CRS is small + { 19, 2, 4, 4, 16}, + // A ConvNet's first layer + { 224, 3, 8, 3, 1 }, + // A ConvNet's first layer with 2x2 convolution, and 1 channel + { 224, 2, 8, 1, 1 }, + // A ConvNet's first layer with 2x2 convolution, and 1 channel, several images in the batch + { 224, 2, 8, 1, 8 }, + // A middle layer of a ConvNet + { 58, 3, 64, 32, 1 }, + // A middle layer of a ConvNet, several images in the batch + { 58, 3, 64, 32, 8 }, + // A deep layer of a ConvNet, several images in the batch + { 16, 3, 256, 128, 8 } + }; + + for (auto kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (auto act_case : cases) { + test_cases.emplace_back(new test_conv_2d( + { act_case[iwh_idx], act_case[iwh_idx], act_case[Cin_idx], act_case[B_idx] }, + { act_case[kwh_idx], act_case[kwh_idx], act_case[Cin_idx], act_case[Cout_idx] }, + kernel_type, 1, 1, 0, 0, 1, 1, false)); + } + } +#endif + + // CONV_2D: + auto calc_conv_output_size = [](int64_t ins, int64_t ks, int s, int p, int d) -> int64_t { + return (ins + 2 * p - d * (ks - 1) - 1) / s + 1; + }; + + //uint32_t s0 = 3; + uint32_t s1 = 5; + uint32_t p0 = 5; + //uint32_t p1 = 2; + uint32_t d0 = 2; + uint32_t d1 = 4; + + for (uint32_t s0 : { 1, 3 }) { + for (uint32_t p1 : { 2, 5 }) { + for (uint32_t Cin : { 1, 25 }) { + for (uint32_t Cout : { 1, 12 }) { + for (uint32_t KH : { 1, 2, 3, 11 }) { + for (uint32_t KW : { 1, 2, 3, 11 }) { + for (uint32_t H : { 1, 133 }) { + for (uint32_t W : { 1, 141 }) { + if (calc_conv_output_size(W, KW, s0, p0, d0) > 0 && + calc_conv_output_size(H, KH, s1, p1, d1) > 0) { + for (auto kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + test_cases.emplace_back(new test_conv_2d( + { W, H, Cin, 2 }, { KW, KH, Cin, Cout }, kernel_type, s0, s1, p0, p1, d0, d1, false)); + } + } + } + } + } + } + } + } + } + } + for (auto kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + test_cases.emplace_back(new test_conv_2d({ 256, 256, 192, 1 }, { 3, 3, 192, 96 }, kernel_type, 1, 1, 1, 1, 1, 1, false)); + } + + // sycl backend will limit task global_range < MAX_INT + // test cases for 2D im2col with large input W and H (occurs in stable-diffusion) + // however these cases need to alloc more memory which may fail in some devices (Intel Arc770, etc.) + // these cases are verified (pass) in Intel(R) Data Center GPU Max 1100 (sycl backend) and NV A30 (cuda backend) + // test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {1024, 1024, 256, 1}, {3, 3, 256, 1}, 1, 1, 1, 1, 1, 1, true)); + // test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {1024, 1024, 256, 1}, {3, 3, 256, 1}, 1, 1, 1, 1, 1, 1, true)); + + test_cases.emplace_back(new test_conv_2d_dw({17, 34, 9, 1}, {3, 3, 1, 9}, GGML_TYPE_F32, 1, 0, 1, false)); + test_cases.emplace_back(new test_conv_2d_dw({17, 34, 9, 1}, {3, 3, 1, 9}, GGML_TYPE_F32, 1, 0, 1, true)); + test_cases.emplace_back(new test_conv_2d_dw({32, 8, 64, 1}, {3, 3, 1, 64}, GGML_TYPE_F32, 2, 1, 1, false)); + test_cases.emplace_back(new test_conv_2d_dw({32, 8, 64, 1}, {3, 3, 1, 64}, GGML_TYPE_F32, 2, 1, 1, true)); + + test_cases.emplace_back(new test_conv_2d_dw({17, 34, 9, 1}, {3, 3, 1, 9}, GGML_TYPE_F16, 1, 0, 1, false)); + test_cases.emplace_back(new test_conv_2d_dw({17, 34, 9, 1}, {3, 3, 1, 9}, GGML_TYPE_F16, 1, 0, 1, true)); + test_cases.emplace_back(new test_conv_2d_dw({32, 8, 64, 1}, {3, 3, 1, 64}, GGML_TYPE_F16, 2, 1, 1, false)); + test_cases.emplace_back(new test_conv_2d_dw({32, 8, 64, 1}, {3, 3, 1, 64}, GGML_TYPE_F16, 2, 1, 1, true)); + + // CONV_3D + auto calc_conv_output_size_3d = [](int64_t ins, int64_t ks, int s, int p, int d) -> int64_t { + return (ins + 2 * p - d * (ks - 1) - 1) / s + 1; + }; + + for (ggml_type kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (int N : {1, 2}) { + for (int IC : {1, 3}) { + for (int OC : {1, 4}) { + for (int s0 : {1, 2}) { + for (int p1 : {0, 1}) { + for (int d2 : {1, 2}) { + int64_t IW = 20, IH = 22, ID = 18; + int64_t KW = 3, KH = 3, KD = 3; + int s1 = s0, s2 = s0; + int p0 = p1, p2 = p1; + int d0 = d2, d1 = d2; + + if (calc_conv_output_size_3d(IW, KW, s0, p0, d0) <= 0 || + calc_conv_output_size_3d(IH, KH, s1, p1, d1) <= 0 || + calc_conv_output_size_3d(ID, KD, s2, p2, d2) <= 0) { + continue; + } + test_cases.emplace_back(new test_conv_3d( + N, IC, ID, IH, IW, + OC, KD, KH, KW, + s0, s1, s2, p0, p1, p2, d0, d1, d2, + kernel_type)); + + // Asymmetric kernel and params + int64_t asym_KW = 5, asym_KH = 1, asym_KD = 3; + int asym_s0 = 2, asym_s1 = 1, asym_s2 = 1; + int asym_p0 = 2, asym_p1 = 0, asym_p2 = 1; + int asym_d0 = 1, asym_d1 = 1, asym_d2 = 2; + + if (calc_conv_output_size_3d(IW, asym_KW, asym_s0, asym_p0, asym_d0) <= 0 || + calc_conv_output_size_3d(IH, asym_KH, asym_s1, asym_p1, asym_d1) <= 0 || + calc_conv_output_size_3d(ID, asym_KD, asym_s2, asym_p2, asym_d2) <= 0) { + continue; + } + test_cases.emplace_back(new test_conv_3d( + N, IC, ID, IH, IW, + OC, asym_KD, asym_KH, asym_KW, + asym_s0, asym_s1, asym_s2, asym_p0, asym_p1, asym_p2, asym_d0, asym_d1, asym_d2, + kernel_type)); + } + } + } + } + } + } + // Case with kernel size 1 + test_cases.emplace_back(new test_conv_3d(1, 4, 8, 8, 8, 8, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + } + + for(uint32_t Cout : {1, 9}){ + for(uint32_t Cin : {1, 7}){ + for(uint32_t K : {1, 3, 1337}){ + for(uint32_t L : {1, 2, 13}){ + for(uint32_t s0: {1, 2, 3}){ + test_cases.emplace_back(new test_conv_transpose_1d({L,Cin,1,1}, {K,Cout,Cin,1}, s0, 0, 1)); + } + } + } + } + } + + test_cases.emplace_back(new test_conv_transpose_1d()); + test_cases.emplace_back(new test_conv_transpose_1d({3,2,1,1}, {2,3,2,1}, 3, 0, 1)); + test_cases.emplace_back(new test_conv_transpose_1d({3,2,1,1}, {2,3,2,1}, 2, 0, 1)); + test_cases.emplace_back(new test_conv_transpose_1d({3,2,1,1}, {2,3,2,1}, 1, 0, 1)); + test_cases.emplace_back(new test_conv_transpose_1d({3,2,1,1}, {3,2,2,1}, 2, 0, 1)); + test_cases.emplace_back(new test_conv_transpose_1d({3,2,1,1}, {3,2,2,1}, 1, 0, 1)); + test_cases.emplace_back(new test_conv_transpose_1d({3,2,1,1}, {3,1,2,1}, 1, 0, 1)); + test_cases.emplace_back(new test_conv_transpose_1d({2,1,1,1}, {3,1,1,1}, 1, 0, 1)); + + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) { + // ConvTranspose1d expressed as mul_mat + col2im (DAC decoder upsampling) + test_cases.emplace_back(new test_col2im_1d(type, 16, 32, 197, 8, 0)); // kernel = 2*stride + test_cases.emplace_back(new test_col2im_1d(type, 4, 3, 7, 2, 0)); + test_cases.emplace_back(new test_col2im_1d(type, 1, 5, 13, 1, 0)); // stride 1, no overlap + test_cases.emplace_back(new test_col2im_1d(type, 6, 4, 11, 3, 1)); // with cropping + test_cases.emplace_back(new test_col2im_1d(type, 2, 3, 9, 3, 0)); // kernel < stride, gap positions are zeroed + test_cases.emplace_back(new test_col2im_1d(type, 5, 4, 11, 2, 0)); // kernel not a multiple of stride, alternating overlap + test_cases.emplace_back(new test_col2im_1d(type, 8, 4, 13, 4, 2)); // padding = stride/2 (DAC causal cropping) + test_cases.emplace_back(new test_col2im_1d(type, 4, 3, 1, 2, 0)); // single column, pure kernel unfold + test_cases.emplace_back(new test_col2im_1d(type, 16, 1, 197, 8, 0)); // OC = 1, mono output stage + test_cases.emplace_back(new test_col2im_1d(type, 1, 5, 13, 3, 0)); // K = 1 with stride > 1, sparse scatter + test_cases.emplace_back(new test_col2im_1d(type, 8, 2, 3, 2, 5)); // cropping eats most of the signal, T_out = 2 + } + + for (ggml_type kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + test_cases.emplace_back(new test_conv_transpose_2d({3, 2, 3, 1}, {2, 2, 1, 3}, 1, kernel_type)); + test_cases.emplace_back(new test_conv_transpose_2d({10, 10, 9, 1}, {3, 3, 1, 9}, 2, kernel_type)); + test_cases.emplace_back(new test_conv_transpose_2d({129, 63, 35, 1}, {3, 3, 48, 35}, 1, kernel_type)); + } + + test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 500, 1, 1})); + test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 5000, 1, 1})); + + test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32, 1, 1, 1})); + test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32, 513, 1, 1})); + test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {100, 10, 1, 1})); + test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {1024, 10, 1, 1})); + test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {1024, 12, 1, 1})); + test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {2000, 10, 1, 1})); + test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {5438, 3, 1, 1})); + + for (int ne3 : {1, 3}) { // CUDA backward pass only supports ne3 == 1 + test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 1, 1})); + test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {2, 1, 1, 1})); + test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 2, 1, 1})); + test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 2, 1})); + test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 1, 2})); + test_cases.emplace_back(new test_repeat(GGML_TYPE_I32, {10, 5, 4, ne3}, {2, 1, 1, 1})); + test_cases.emplace_back(new test_repeat(GGML_TYPE_I16, {10, 5, 4, ne3}, {1, 1, 1, 2})); + test_cases.emplace_back(new test_repeat(GGML_TYPE_BF16, {10, 5, 4, ne3}, {2, 1, 1, 1})); + } + + for (bool view : {false, true}) { + test_cases.emplace_back(new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {1, 1, 1, 1}, view)); + test_cases.emplace_back(new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {2, 1, 1, 1}, view)); + test_cases.emplace_back(new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {1, 2, 1, 1}, view)); + test_cases.emplace_back(new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {1, 1, 2, 1}, view)); + test_cases.emplace_back(new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {1, 1, 1, 2}, view)); + } + + test_cases.emplace_back(new test_dup(GGML_TYPE_F32)); + test_cases.emplace_back(new test_dup(GGML_TYPE_F16)); + test_cases.emplace_back(new test_dup(GGML_TYPE_I32)); + test_cases.emplace_back(new test_dup(GGML_TYPE_I16)); + test_cases.emplace_back(new test_dup(GGML_TYPE_F32, {10, 10, 5, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_dup(GGML_TYPE_F16, {10, 10, 5, 1}, {0, 2, 1, 3})); // dup by rows + test_cases.emplace_back(new test_dup(GGML_TYPE_F32, {10, 10, 5, 1}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_dup(GGML_TYPE_F16, {10, 10, 5, 1}, {1, 0, 2, 3})); // dup dst not-contiguous + test_cases.emplace_back(new test_dup(GGML_TYPE_I16, {10, 8, 3, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_dup(GGML_TYPE_I16, {10, 8, 3, 1}, {1, 2, 0, 3})); + + for (int dim = 1; dim < GGML_MAX_DIMS; ++dim) { + test_cases.emplace_back(new test_set(GGML_TYPE_F32, GGML_TYPE_F32, {6, 5, 4, 3}, dim, false)); + test_cases.emplace_back(new test_set(GGML_TYPE_F32, GGML_TYPE_F32, {6, 5, 4, 3}, dim, true)); + } + + for (int dim = 1; dim < GGML_MAX_DIMS; ++dim) { + test_cases.emplace_back(new test_set(GGML_TYPE_I32, GGML_TYPE_I32, {6, 5, 4, 3}, dim, false)); + test_cases.emplace_back(new test_set(GGML_TYPE_I32, GGML_TYPE_I32, {6, 5, 4, 3}, dim, true)); + } + + // same-type copy + for (ggml_type type : all_types) { + const auto nk = ggml_blck_size(type); + + for (int k = 1; k < 4; ++k) { + test_cases.emplace_back(new test_cpy(type, type, {k*nk, 2, 3, 4})); + test_cases.emplace_back(new test_cpy(type, type, {k*nk, 2, 3, 4}, {-1,-1,-1,-1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_cpy(type, type, {k*nk, 2, 3, 4}, {-1,-1,-1,-1}, {0, 3, 1, 2}, {0, 2, 1, 3})); + } + } + + for (ggml_type type_src : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_F32}) { + for (ggml_type type_dst : all_types) { + test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 4, 4, 4})); + test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // cpy by rows + } + } + for (ggml_type type_src : all_types) { + for (ggml_type type_dst : {GGML_TYPE_F32}) { + test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 4, 4, 4})); + test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // cpy by rows + } + } + for (ggml_type type_src : {GGML_TYPE_F16, GGML_TYPE_F32}) { + for (ggml_type type_dst : {GGML_TYPE_F16, GGML_TYPE_F32}) { + test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3})); // cpy not-contiguous + } + } + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_I32, {256, 2, 3, 4})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_I32, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_F32, {256, 2, 3, 4})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_F32, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {256, 4, 3, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 3, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 3, 3}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); + test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {256, 4, 3, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {256, 4, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); + test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {256, 4, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); + test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_I32, {256, 4, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); + test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_I32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {2, 2097121, 1, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {2, 2, 524281, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {128, 2, 3, 1}, {128, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {128, 4, 3, 1})); // strided dst + test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {128, 2, 3, 1}, {128, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {128, 4, 3, 1})); // strided dst + + // CPY - different src/dst shapes (reshaping via CPY) + // Use permutations of {3, 5, 7, 32}. Total elements: 3*5*7*32 = 3360. + // Each src permutation is tested against canonical sorted and reverse dst (skip self). + { + std::array dims = {3, 5, 7, 32}; + std::sort(dims.begin(), dims.end()); + std::array canonical = dims; + std::array reversed = {32, 7, 5, 3}; + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + std::array cur = dims; + do { + if (cur != canonical) { + test_cases.emplace_back(new test_cpy(type, type, cur, canonical)); + } + if (cur != reversed) { + test_cases.emplace_back(new test_cpy(type, type, cur, reversed)); + } + if (cur[0] == 32 && type == GGML_TYPE_F32) { + if (canonical[0] == 32) { + test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, cur, canonical)); + } + if (reversed[0] == 32) { + test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, cur, reversed)); + } + } + std::next_permutation(cur.begin(), cur.end()); + } while (cur != canonical); + } + } + + for (ggml_type type_dst : { GGML_TYPE_F32, GGML_TYPE_I32, GGML_TYPE_F16, GGML_TYPE_BF16 }) { + for (bool use_view_slice : { true, false }) { + for (std::array ne : std::initializer_list>{ {2, 1, 1, 1}, {2, 1, 3, 5}, + {2, 3, 5, 7}, {1, 4, 4, 1}, {1, 8, 17, 1}, {10, 10, 10, 1} }) { + if (use_view_slice && (type_dst == GGML_TYPE_F16 || type_dst == GGML_TYPE_BF16)) { + continue; // TODO: add after WebGPU is fixed + } + test_cases.emplace_back(new test_cont(type_dst, ne, use_view_slice)); + } + } + } + + auto add_test_bin_bcast = [&](ggml_type type, std::array ne, std::array nr, bool perm1 = false, bool src_overlap = false) { + for (auto op : {ggml_add, ggml_sub, ggml_mul, ggml_div}) { + test_cases.emplace_back(new test_bin_bcast(op, type, ne, nr, 1, perm1, src_overlap)); + } + }; + for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { + for (bool perm1 : {false, true}) { + add_test_bin_bcast(type, {1, 1, 8, 1}, {1, 1, 1, 1}, perm1); + add_test_bin_bcast(type, {1, 1, 1, 1}, {32, 1, 1, 1}, perm1); + add_test_bin_bcast(type, {1, 1, 320, 320}, {1, 1, 1, 1}, perm1); + add_test_bin_bcast(type, {10, 5, 1, 1}, {1, 1, 1, 1}, perm1); + add_test_bin_bcast(type, {10, 5, 4, 1}, {1, 1, 1, 1}, perm1); + add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 1, 1, 1}, perm1); + add_test_bin_bcast(type, {10, 5, 4, 3}, {2, 1, 1, 1}, perm1); + add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 2, 1, 1}, perm1); + add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 1, 2, 1}, perm1); + add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 1, 1, 2}, perm1); + add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 1, 2, 2}, perm1); + add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 2, 2, 2}, perm1); + add_test_bin_bcast(type, {10, 5, 4, 3}, {2, 2, 2, 2}, perm1); + } + + // src_overlap + add_test_bin_bcast(type, {10, 5, 4, 6}, {1, 1, 1, 1}, false, true); + add_test_bin_bcast(type, {10, 5, 4, 5}, {1, 1, 1, 1}, false, true); + add_test_bin_bcast(type, {1, 1, 120, 120}, {1, 1, 1, 1}, false, true); + add_test_bin_bcast(type, {1, 1, 4, 320}, {1, 1, 1, 1}, false, true); + + // test case for k_bin_bcast_unravel in CUDA backend + add_test_bin_bcast(type, {1, 1, 65536, 1}, {256, 1, 1, 1}); + + // stable diffusion + add_test_bin_bcast(type, {1280, 1, 1, 1}, {1, 1, 1, 1}); + add_test_bin_bcast(type, {1280, 1, 1, 1}, {1, 16, 16, 1}); + add_test_bin_bcast(type, {1280, 16, 16, 1}, {1, 1, 1, 1}); + add_test_bin_bcast(type, {1280, 1, 1, 1}, {1, 256, 1, 1}); + add_test_bin_bcast(type, {1, 1, 1280, 1}, {16, 16, 1, 1}); + add_test_bin_bcast(type, {16, 16, 1280, 1}, {1, 1, 1, 1}); + add_test_bin_bcast(type, {1, 1, 1920, 1}, {16, 16, 1, 1}); + add_test_bin_bcast(type, {1, 1, 2560, 1}, {16, 16, 1, 1}); + add_test_bin_bcast(type, {1, 1, 1280, 1}, {32, 32, 1, 1}); + add_test_bin_bcast(type, {1, 1, 1920, 1}, {32, 32, 1, 1}); + add_test_bin_bcast(type, {1, 1, 640, 1}, {32, 32, 1, 1}); + add_test_bin_bcast(type, {5120, 1, 1, 1}, {1, 256, 1, 1}); + add_test_bin_bcast(type, {640, 1, 1, 1}, {1, 1, 1, 1}); + add_test_bin_bcast(type, {64, 262144, 1, 1}, {1, 1, 1, 1}); + //add_test_bin_bcast(type, {3, 3, 2560, 1280}, {1, 1, 1, 1}); + //add_test_bin_bcast(type, {3, 3, 2560, 1280}, {2, 1, 1, 1}); + } + + // single inplace tests, especially important for WebGPU backend since kernels for inplace vs. not are different + test_cases.emplace_back(new test_bin_bcast(ggml_add_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16)); + test_cases.emplace_back(new test_bin_bcast(ggml_mul_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16)); + test_cases.emplace_back(new test_bin_bcast(ggml_sub_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16)); + test_cases.emplace_back(new test_bin_bcast(ggml_div_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16)); + + // fusion + test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {2, 1, 1, 1}, 2)); + test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 2, 1, 1}, 3)); + test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {1, 1, 2, 1}, 4)); + test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 2}, 5)); + test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {1, 1, 2, 2}, 6)); + test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {1, 2, 2, 2}, 7)); + test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {2, 2, 2, 2}, 8)); + test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16)); + + test_cases.emplace_back(new test_scale()); + test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f)); + test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, true)); // inplace test + test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {100, 10, 10, 10}, 2.0f, 1.0f)); + test_cases.emplace_back(new test_softcap(GGML_TYPE_F32, {10, 10, 10, 10}, 50.0f)); + test_cases.emplace_back(new test_silu_back()); + + for (float eps : { 0.0f, 1e-6f, 1e-4f, 1e-1f, 10.f }) { + for (uint32_t n : { 64, 1025 }) { + for (bool v : { false, true }) { + test_cases.emplace_back(new test_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, v, eps)); + test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, v, eps)); + } + test_cases.emplace_back(new test_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, false, eps, true)); + test_cases.emplace_back(new test_rms_norm_back(GGML_TYPE_F32, { n, 5, 4, 3 }, eps)); + test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false)); + test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true)); + test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false, true)); + } + } + + // in-place tests + test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {64, 5, 4, 3}, false, 1e-6f, true)); + + for (float eps : { 0.0f, 1e-6f, 1e-4f, 1e-1f, 1.0f }) { + for (uint32_t n : { 64, 1025 }) { + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false)); + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true)); + test_cases.emplace_back(new test_norm_mul_add(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false)); + test_cases.emplace_back(new test_norm_mul_add(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true)); + test_cases.emplace_back(new test_add_rms_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false)); + test_cases.emplace_back(new test_add_rms_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true)); + } + } + for (uint32_t n : {1, 511, 1025, 8192, 33*512}) { + for (bool multi_add : {false, true}) { + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {n, 1, 1, 1}, 1e-6f, false, multi_add)); + } + test_cases.emplace_back(new test_add_rms_norm(GGML_TYPE_F32, {n, 1, 1, 1}, 1e-6f, false)); + } + + for (auto multi_add : {false, true}) { + for (auto set_rows : {false, true}) { + for (auto rope : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) { + test_cases.emplace_back(new test_rms_norm_mul_rope({768, 1, 1, 1}, 1e-6f, multi_add, set_rows, rope)); + test_cases.emplace_back(new test_rms_norm_mul_rope({768, 3, 1, 1}, 1e-6f, multi_add, set_rows, rope)); + test_cases.emplace_back(new test_rms_norm_mul_rope({768, 3, 5, 1}, 1e-6f, multi_add, set_rows, rope)); + test_cases.emplace_back(new test_rms_norm_mul_rope({128, 32, 2, 1}, 1e-6f, multi_add, set_rows, rope)); + test_cases.emplace_back(new test_rms_norm_mul_rope({128, 4, 2, 1}, 1e-6f, multi_add, set_rows, rope)); + test_cases.emplace_back(new test_rms_norm_mul_rope({128, 32, 50, 1}, 1e-6f, multi_add, set_rows, rope)); + test_cases.emplace_back(new test_rms_norm_mul_rope({128, 4, 50, 1}, 1e-6f, multi_add, set_rows, rope)); + test_cases.emplace_back(new test_rms_norm_mul_rope({8192, 2, 2, 1}, 1e-6f, multi_add, set_rows, rope)); + test_cases.emplace_back(new test_rms_norm_mul_rope({8192, 2, 2, 1}, 1e-6f, multi_add, set_rows, rope)); + } + } + } + for (int64_t d_conv : {3, 4, 9}) { + for (int64_t d_inner: {1024, 1536, 2048}) { + test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1})); + test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {2 * d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1})); + test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {d_conv, d_inner, 4, 1}, {d_conv, d_inner, 1, 1})); + // long token (n_t > 32, exercises the long_token kernel path) + test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {d_conv - 1 + 64, d_inner, 1, 1}, {d_conv, d_inner, 1, 1})); + test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {d_conv - 1 + 64, d_inner, 4, 1}, {d_conv, d_inner, 1, 1})); + } + } + + // fused ssm_conv + (optional) bias_add + silu. The bias-only graph (no silu) is intentionally + // not tested since there's no fusion for that pattern in ggml_cuda_can_fuse. + for (int64_t d_conv : {3, 4, 9}) { + for (int64_t d_inner : {1024, 1536, 2048}) { + for (bool fuse_bias : {false, true}) { + // short token path (n_t <= 32) + test_cases.emplace_back(new test_ssm_conv_bias_silu( + GGML_TYPE_F32, {d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1}, fuse_bias)); + test_cases.emplace_back(new test_ssm_conv_bias_silu( + GGML_TYPE_F32, {2 * d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1}, fuse_bias)); + test_cases.emplace_back(new test_ssm_conv_bias_silu( + GGML_TYPE_F32, {d_conv, d_inner, 4, 1}, {d_conv, d_inner, 1, 1}, fuse_bias)); + // long token path (n_t > 32) + test_cases.emplace_back(new test_ssm_conv_bias_silu( + GGML_TYPE_F32, {d_conv - 1 + 64, d_inner, 1, 1}, {d_conv, d_inner, 1, 1}, fuse_bias)); + test_cases.emplace_back(new test_ssm_conv_bias_silu( + GGML_TYPE_F32, {d_conv - 1 + 64, d_inner, 4, 1}, {d_conv, d_inner, 1, 1}, fuse_bias)); + } + } + } + + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 16, 1, 1024, 1, 32, 4)); // Mamba-1 + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 16, 2, 32, 4)); // Mamba-2 + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 256, 64, 8, 2, 32, 4)); // Falcon-H1 + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 128, 4, 4, 16, 2, true)); // x/B/C overlap + + test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 1, 1)); + test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 32, 1)); + test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 32, 4)); + test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 128, 4)); + + test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 1, 1)); + test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 32, 1)); + test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 32, 4)); + test_cases.emplace_back(new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 128, 4)); + + test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 1, 1)); + test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 32, 1)); + test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 32, 4)); + test_cases.emplace_back(new test_gla(GGML_TYPE_F32, 32, 64, 128, 4)); + + // FWHT tests + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 1, 128)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 64, 1, 64)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 256, 1, 256)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 512, 1, 512)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 32, 128)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 4, 128, {2, 3})); + +#if 0 + // > 4GB A matrix. Too slow to be enabled by default. + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 900000, 3, 2592, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1700000, 96, 2592, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1700000, 3, 2592, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1700000, 1, 2592, {1, 1}, {1, 1})); + + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 128, 128, false, 8192, 2, 5120)); // Llama-4-Maverick-17B-128E-PAB-Q8_0 + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 128, 128, false, 8192, 1, 5120)); // Llama-4-Maverick-17B-128E-PAB-Q8_0 + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8192, 1, 5120, {128, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8192, 512, 5120, {128, 1}, {1, 1})); +#endif + + for (ggml_type type_a : all_types) { + for (int i = 1; i < 10; ++i) { + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, i, 256, { 1, 1}, {1, 1})); + } + } + + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); + + +#if 0 + { + // Test paths in OpenCL + std::vector ns = {32, 64, 128, 256, 512, 1024, 4096}; + std::vector ks = {896, 1536, 4096}; + for (auto n : ns) { + for (auto k : ks) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 1024, n, k, {1, 1}, {1, 1})); + } + } + } +#endif + +#if 1 + for (ggml_type type_a : base_types) { + for (ggml_type type_b : {GGML_TYPE_F32, GGML_TYPE_F16}) { + std::vector ks = { 256 }; + if (ggml_blck_size(type_a) == 1) { + ks.push_back(4); + } + for (auto k : ks) { + // test cases without permutation + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {1, 1}, {2, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {1, 1}, {1, 2})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {3, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {3, 1}, {2, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {3, 2}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {3, 2}, {2, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {3, 2}, {1, 2})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {3, 2}, {2, 2})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 4, k, {3, 2}, {2, 2})); + + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {1, 1}, {2, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {1, 1}, {1, 2})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {3, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {3, 1}, {2, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {3, 2}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {3, 2}, {2, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {3, 2}, {1, 2})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {3, 2}, {2, 2})); + + // test cases with permutation + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {2, 3}, {1, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {2, 3}, {1, 1}, {0, 1, 3, 2})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, k, {2, 3}, {1, 1}, {0, 3, 2, 1})); + + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 4, k, {2, 3}, {1, 1}, {0, 3, 2, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 8, k, {2, 3}, {1, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 8, k, {2, 3}, {1, 1}, {0, 1, 3, 2})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 8, k, {2, 3}, {1, 1}, {0, 3, 2, 1})); + + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {2, 3}, {1, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {2, 3}, {1, 1}, {0, 1, 3, 2})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, k, {2, 3}, {1, 1}, {0, 3, 2, 1})); + } + + // test cases with large ne00/ne10 to cover stream-k fixup + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, 1024, {3, 2}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 8, 1024, {3, 2}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 16, 1024, {3, 2}, {1, 1})); + + // test cases with large batch size + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 8, 256, {1536, 1}, {1, 1})); + } + } + + // BF16 is absent from base_types: add the 3 standard non-contig permutations explicitly + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 1, 256, {2, 3}, {1, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 1, 256, {2, 3}, {1, 1}, {0, 1, 3, 2})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 1, 256, {2, 3}, {1, 1}, {0, 3, 2, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 8, 256, {2, 3}, {1, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 8, 256, {2, 3}, {1, 1}, {0, 1, 3, 2})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 8, 256, {2, 3}, {1, 1}, {0, 3, 2, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 1, 3, 2})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 3, 2, 1})); + + for (ggml_type type_a : other_types) { + for (ggml_type type_b : {GGML_TYPE_F32}) { + if (ggml_blck_size(type_a) != 256) { + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, ggml_blck_size(type_a), {1, 1}, {1, 1})); + } + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, 256, {1, 1}, {1, 1})); + } + } +#else + // m = a rows + // n = b rows + // k = cols + std::uniform_int_distribution<> dist_m(1, 128); + std::uniform_int_distribution<> dist_n(16, 128); + std::uniform_int_distribution<> dist_k(1, 16); + for (int i = 0; i < 1000; i++) { + for (ggml_type type_a : all_types) { + for (ggml_type type_b : {GGML_TYPE_F32}) { + int m = dist_m(rng); + int n = dist_n(rng); + int k = dist_k(rng) * ggml_blck_size(type_a); + test_cases.emplace_back(new test_mul_mat(type_a, type_b, m, n, k, { 1, 1}, {1, 1})); + } + } + } +#endif + + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 2, 128, { 8, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 83, 2, 128, { 8, 1}, {4, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 2, 64, { 8, 1}, {4, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 83, 2, 64, { 8, 1}, {4, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 45, 128, { 8, 1}, {4, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 128, 45, 64, { 8, 1}, {4, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1056, 1, 193, {1, 1}, {4, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1056, 1, 67, {1, 1}, {4, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 16, 32, 32, { 1, 1}, {1, 1}, {0, 1, 2, 3}, 64, 3)); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 77, 77, {12,1}, {1,1})); + + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 576, 512, 576, {1,1}, {1,1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 1, 2048, 8192, {1, 1}, {1, 1})); + for (ggml_type type_a : all_types) { + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1, 64, 256, {1, 1}, {1, 1})); + } + + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 6, 4096, 5120, {1, 1}, {1, 1})); + +#if 0 + // test the mat-mat path for Metal + for (int k = 1; k < 512; ++k) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 127, k, {12,1}, {1,1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 127, k, {12,1}, {1,1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 77, k, {12,1}, {1,1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 77, k, {12,1}, {1,1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 128, k, {12,1}, {1,1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 128, k, {12,1}, {1,1})); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, false, 50, 200, k)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, true, 50, 200, k)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 16, 16, false, 50, 200, k)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 16, 16, true, 50, 200, k)); + } +#endif + + for (auto bs2 : {1,3}) { + for (auto bs : {1,2,4,8}) { + for (auto nr : {1,4}) { + for (uint32_t m = 0; m < 2; ++m) { + for (uint32_t k = 0; k < 2; ++k) { + for (ggml_type type: {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_F32}) { + test_cases.emplace_back(new test_mul_mat(type, GGML_TYPE_F32, 1056 + m, 1, 128 + k, {bs, bs2}, {nr, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(type, GGML_TYPE_F32, 128 + m, 1, 1056 + k, {bs, bs2}, {nr, 1}, {0, 1, 2, 3}, 2*1056 + k)); + } + } + } + } + } + } + + // sycl backend will limit task global_range < MAX_INT + // test case for f16-type-convert-to-fp32 kernel with large k under fp32 compute dtype (occurs in stable-diffusion) + // however this case needs to alloc more memory which may fail in some devices (Intel Arc770, etc.) + // this case is verified (pass) in Intel(R) Data Center GPU Max 1100 (sycl backend) and NV A30 (cuda backend) + // test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 512, 262144, 9216, {1, 1}, {1, 1})); + + // test large experts*tokens + for (bool b : {false, true}) { + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, b, 32, 1024, 16)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 2, 2, b, 32, 8192, 64)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, b, 50, 200, 64)); + } + + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 1, 1, false, 8, 16, 1)); + test_cases.emplace_back(new test_mul_mat_id_fusion(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, false, 32, 32, 32, 3)); + + // gpt-oss issue with Vulkan mmq_id + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_MXFP4, GGML_TYPE_F32, 32, 2, false, 2880, 32, 2880)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_0, GGML_TYPE_F32, 32, 2, false, 2880, 32, 2880)); + + for (ggml_type type_a : all_types) { + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 2, false, 64, 16, 3*ggml_blck_size(type_a))); + } + + for (ggml_type type_a : base_types) { + for (ggml_type type_b : {GGML_TYPE_F32 /*, GGML_TYPE_F16 */}) { + for (int n_mats : {4, 8}) { + for (int n_used : {1, 2, 4}) { + for (bool b : {false, true}) { + for (int n : {1, 4, 5, 17, 32, 129}) { + int m = 512; + int k = 256; + test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, n_mats, n_used, b, m, n, k)); + } + } + } + } + } + } + + for (ggml_type type_a : other_types) { + for (ggml_type type_b : {GGML_TYPE_F32 /*, GGML_TYPE_F16 */}) { + for (int n_mats : {4}) { + for (int n_used : {2}) { + for (bool b : {false}) { + for (int n : {1, 32}) { + int m = 512; + int k = 256; + test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, n_mats, n_used, b, m, n, k)); + } + } + } + } + } + } + + for (int bs : {1, 4, 512}) { + for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_K}) { + for (ggml_type type_b : {GGML_TYPE_F32}) { + // test with mul after (ffn_moe_weighted) + test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, type_b, 128, 8, false, 768, bs, 2048, 1, true)); + } + } + } + + for (ggml_type type_a : base_types) { + for (ggml_type type_b : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (int n : {1, 16}) { + for (int k : {1, 16}) { + for (int bs2 : {1, 3}) { + for (int bs3 : {1, 3}) { + for (int nr2 : {1, 2}) { + for (int nr3 : {1, 2}) { + test_cases.emplace_back(new test_out_prod(type_a, type_b, 256, n, k, {bs2, bs3}, {nr2, nr3})); + } + } + } + } + } + } + } + } + + // ne2 sweep to cover the cublasSgemmStridedBatched path (dps2 == 1, ne2 > 1) + for (int64_t ne2 : {1, 8, 16, 32}) { + test_cases.emplace_back(new test_out_prod(GGML_TYPE_F32, GGML_TYPE_F32, + 256, 16, 16, {ne2, 1}, {1, 1})); + } + + // nr2 sweep to cover the cublasSgemmBatched pointer-array path (dps2 > 1) + for (int64_t nr2 : {8, 16, 32}) { + test_cases.emplace_back(new test_out_prod(GGML_TYPE_F32, GGML_TYPE_F32, + 256, 16, 16, {1, 1}, {nr2, 1})); + } + + // add_id + for (ggml_type type_a : {GGML_TYPE_F32}) { + for (ggml_type type_b : {GGML_TYPE_F32}) { + for (int n_mats : {4, 8}) { + for (int n_used : {1, 2, 4}) { + for (int n_embd : {32, 129}) { + for (int n_token : {1, 32, 129}) { + test_cases.emplace_back(new test_add_id(type_a, type_b, n_embd, n_mats, n_used, n_token)); + } + } + } + } + } + } + + for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { + test_cases.emplace_back(new test_sqr (type)); + test_cases.emplace_back(new test_sqrt (type)); + test_cases.emplace_back(new test_log (type)); + test_cases.emplace_back(new test_sin (type)); + test_cases.emplace_back(new test_cos (type)); + test_cases.emplace_back(new test_clamp (type)); + test_cases.emplace_back(new test_leaky_relu(type)); + test_cases.emplace_back(new test_floor (type)); + test_cases.emplace_back(new test_ceil (type)); + test_cases.emplace_back(new test_round (type)); + test_cases.emplace_back(new test_trunc (type)); + test_cases.emplace_back(new test_sqr (type, {7, 1, 5, 3})); + test_cases.emplace_back(new test_sqr (type, {1024, 1024, 1, 1})); + test_cases.emplace_back(new test_sqrt (type, {7, 1, 5, 3})); + test_cases.emplace_back(new test_sqrt (type, {1024, 1024, 1, 1})); + test_cases.emplace_back(new test_log (type, {7, 1, 5, 3})); + test_cases.emplace_back(new test_log (type, {1024, 1024, 1, 1})); + test_cases.emplace_back(new test_sin (type, {7, 1, 5, 3})); + test_cases.emplace_back(new test_sin (type, {1024, 1024, 1, 1})); + test_cases.emplace_back(new test_cos (type, {7, 1, 5, 3})); + test_cases.emplace_back(new test_cos (type, {1024, 1024, 1, 1})); + test_cases.emplace_back(new test_clamp (type, {7, 1, 5, 3})); + test_cases.emplace_back(new test_clamp (type, {1024, 1024, 1, 1})); + test_cases.emplace_back(new test_leaky_relu(type, {7, 1, 5, 3})); + test_cases.emplace_back(new test_leaky_relu(type, {1024, 1024, 1, 1})); + test_cases.emplace_back(new test_floor (type, {7, 1, 5, 3})); + test_cases.emplace_back(new test_floor (type, {1024, 1024, 1, 1})); + test_cases.emplace_back(new test_ceil (type, {7, 1, 5, 3})); + test_cases.emplace_back(new test_ceil (type, {1024, 1024, 1, 1})); + test_cases.emplace_back(new test_round (type, {7, 1, 5, 3})); + test_cases.emplace_back(new test_round (type, {1024, 1024, 1, 1})); + test_cases.emplace_back(new test_trunc (type, {7, 1, 5, 3})); + test_cases.emplace_back(new test_trunc (type, {1024, 1024, 1, 1})); + } + + test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 1, 1}, 5)); + test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 3, 1}, 5)); + test_cases.emplace_back(new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 3, 2}, 5)); + +#if 0 + std::uniform_int_distribution<> dist_ne1(1, 50); + int exponent = 1; + while (exponent < (1 << 17)) { + std::uniform_int_distribution<> dist_ne0(exponent, 2*exponent); + + for (int n = 0; n < 10; ++n) { + int64_t ne0 = dist_ne0(rng); + int64_t ne1 = dist_ne1(rng); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, GGML_TYPE_F32, {ne0, ne1, 1, 1}, n/2 == 0, 0.1f, ne0 < 1000 ? 4.0f : 0.0f)); + } + + exponent <<= 1; + } +#endif + for (bool mask : {false, true}) { + for (bool sinks : {false, true}) { + for (float max_bias : {0.0f, 8.0f}) { + if (!mask && max_bias > 0.0f) continue; + for (float scale : {1.0f, 0.1f}) { + for (int64_t ne0 : {16, 1024}) { + for (int64_t ne1 : {16, 1024}) { + if (mask) { + for (ggml_type m_prec : {GGML_TYPE_F32, GGML_TYPE_F16}) { + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0, ne1, 1, 1}, mask, sinks, m_prec, {1, 1}, scale, max_bias)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0-1, ne1-1, 1, 1}, mask, sinks, m_prec, {1, 1}, scale, max_bias)); + + if (ne0 <= 32 && ne1 <= 32) { + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0, ne1, 1, 3}, mask, sinks, m_prec, {3, 1}, scale, max_bias)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0-1, ne1-1, 1, 1}, mask, sinks, m_prec, {2, 3}, scale, max_bias)); + } + } + } else { + /* The precision of mask here doesn't matter as boolean mask is false */ + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0, ne1, 1, 1}, mask, sinks, GGML_TYPE_F32, {1, 1}, scale, max_bias)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {ne0-1, ne1-1, 1, 1}, mask, sinks, GGML_TYPE_F32, {1, 1}, scale, max_bias)); + } + } + } + } + } + // inplace tests + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, mask, sinks, GGML_TYPE_F32, {1, 1}, 0.1f, 0.0f, true)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, mask, sinks, GGML_TYPE_F16, {1, 1}, 0.1f, 0.0f, true)); + } + } + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, true, true, GGML_TYPE_F32, {1, 1}, 0.1f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, true, false, GGML_TYPE_F16, {1, 1}, 0.1f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, false, true, GGML_TYPE_F32, {1, 1}, 0.1f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, true, GGML_TYPE_F32, {1, 1}, 0.1f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, false, GGML_TYPE_F16, {1, 1}, 0.1f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, true, GGML_TYPE_F32, {1, 1}, 0.1f, 8.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, true, GGML_TYPE_F16, {1, 1}, 0.1f, 8.0f)); + + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {200001, 2, 3, 1}, true, true, GGML_TYPE_F32, {1, 1}, 0.1f, 8.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {200001, 2, 3, 1}, true, true, GGML_TYPE_F16, {1, 1}, 0.1f, 8.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {200000, 1, 1, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {200000, 4, 1, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {643251, 3, 1, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + + for (float max_bias : {0.0f, 8.0f}) { + for (float scale : {1.0f, 0.1f}) { + for (int64_t ne0 : {16, 1024}) { + for (int64_t ne1 : {16, 1024}) { + test_cases.emplace_back(new test_soft_max_back(GGML_TYPE_F32, {ne0, ne1, 1, 1}, scale, max_bias)); + test_cases.emplace_back(new test_soft_max_back(GGML_TYPE_F32, {ne0-1, ne1-1, 1, 1}, scale, max_bias)); + test_cases.emplace_back(new test_soft_max_back(GGML_TYPE_F32, {ne0, ne1, 2, 3}, scale, max_bias)); + } + } + } + } + + for (bool fw : {true, false}) { // fw == forward + bool all = true; + + for (float fs : { 1.0f, 1.4245f }) { + for (float ef : { 0.0f, 0.7465f }) { + for (float af : { 1.0f, 1.4245f }) { + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (bool ff : {false, true}) { // freq_factors + for (float v : { 0, 1 }) { + test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); // llama 7B + + if (all) { + test_cases.emplace_back(new test_rope(type, {128, 40, 2, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); // llama 13B + test_cases.emplace_back(new test_rope(type, {128, 52, 2, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); // llama 30B + test_cases.emplace_back(new test_rope(type, {128, 64, 2, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); // llama 65B + test_cases.emplace_back(new test_rope(type, {16, 16, 8192, 1}, 16, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); + } + + if (all) { + test_cases.emplace_back(new test_rope(type, { 64, 1, 2, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (falcon 7B) + test_cases.emplace_back(new test_rope(type, { 64, 71, 2, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (falcon 7B) + test_cases.emplace_back(new test_rope(type, { 64, 8, 2, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (falcon 40B) + + test_cases.emplace_back(new test_rope(type, { 80, 32, 2, 1}, 20, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); + test_cases.emplace_back(new test_rope(type, { 80, 32, 2, 1}, 32, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); + test_cases.emplace_back(new test_rope(type, { 80, 32, 4, 1}, 32, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); + + test_cases.emplace_back(new test_rope(type, { 80, 32, 2, 1}, 20, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (stablelm) + test_cases.emplace_back(new test_rope(type, { 80, 32, 2, 1}, 32, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (phi-2) + test_cases.emplace_back(new test_rope(type, { 80, 32, 4, 1}, 32, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (phi-2) + test_cases.emplace_back(new test_rope(type, { 16, 16, 8192, 1}, 16, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); + } + + if (all) { + test_cases.emplace_back(new test_rope(type, {128, 12, 2, 1}, 128, GGML_ROPE_TYPE_MROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen2vl 2B) + test_cases.emplace_back(new test_rope(type, {128, 28, 2, 1}, 128, GGML_ROPE_TYPE_MROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen2vl 7B) + test_cases.emplace_back(new test_rope(type, {128, 12, 2, 1}, 20, GGML_ROPE_TYPE_MROPE, 512, fs, ef, af, ff, v, fw)); + test_cases.emplace_back(new test_rope(type, {128, 28, 2, 1}, 32, GGML_ROPE_TYPE_MROPE, 512, fs, ef, af, ff, v, fw)); + test_cases.emplace_back(new test_rope(type, {128, 12, 2, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,imrope (qwen3vl 2B) + test_cases.emplace_back(new test_rope(type, {128, 28, 2, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,imrope (qwen3vl 7B) + test_cases.emplace_back(new test_rope(type, {128, 12, 2, 1}, 20, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); + test_cases.emplace_back(new test_rope(type, {128, 28, 2, 1}, 32, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); + test_cases.emplace_back(new test_rope(type, { 80, 16, 2, 1}, 80, GGML_ROPE_TYPE_VISION, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen2vl ViT) + test_cases.emplace_back(new test_rope(type, {128, 16, 2, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen3vl) + test_cases.emplace_back(new test_rope(type, {16, 16, 8192, 1}, 16, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); + } + + test_cases.emplace_back(new test_rope(type, { 64, 128, 2, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (falcon 40B) + } + + // build_rope_2d-style: ROPE on a non-contiguous view + // that starts at a non-zero offset along dim 0 + // (e.g. gemma4v vision second-half view). + for (int rmode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE, GGML_ROPE_TYPE_VISION }) { + test_cases.emplace_back(new test_rope(type, { 36, 16, 2457, 1}, 36, rmode, 512, fs, ef, af, ff, 2, fw)); + } + } + + all = false; + } + } + } + } + } + + // single inplace test per type/mode/ff + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE, GGML_ROPE_TYPE_VISION}) { + for (bool ff : {false, true}) { + test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, ff, 0, true, true)); + test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, ff, 1, true, true)); + test_cases.emplace_back(new test_rope(type, {128, 32, 2, 3}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, ff, 1, true, true)); + } + } + } + + for (int v : { 0, 1, 2, 3 }) { + for (int dim : { 0, 1, 2, 3, }) { + test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {11, 12, 13, 14}, 7, dim, v)); + test_cases.emplace_back(new test_concat(GGML_TYPE_F16, {11, 12, 13, 14}, 7, dim, v)); + test_cases.emplace_back(new test_concat(GGML_TYPE_BF16, {11, 12, 13, 14}, 7, dim, v)); + test_cases.emplace_back(new test_concat(GGML_TYPE_I8, {11, 12, 13, 14}, 7, dim, v)); + test_cases.emplace_back(new test_concat(GGML_TYPE_I16, {11, 12, 13, 14}, 7, dim, v)); + test_cases.emplace_back(new test_concat(GGML_TYPE_I32, {11, 12, 13, 14}, 7, dim, v)); + test_cases.emplace_back(new test_concat(GGML_TYPE_I64, {11, 12, 13, 14}, 7, dim, v)); + } + } + + for (ggml_type type_a : { GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 }) { + for (int dim : { 0, 1, 2, 3, }) { + test_cases.emplace_back(new test_concat(type_a, {128, 12, 13, 14}, dim == 0 ? 256 : 7, dim, 0)); + } + } + + for (ggml_sort_order order : {GGML_SORT_ORDER_ASC, GGML_SORT_ORDER_DESC}) { + for (uint32_t i = 4; i <= 1024*1024; i *= 2) { + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {i-1, 1, 1, 1})); + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {i, 1, 1, 1})); + } + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {16, 10, 10, 10}, order)); + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {60, 10, 10, 10}, order)); // qwen + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {1023, 2, 1, 3}, order)); + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {1024, 2, 1, 3}, order)); + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {1025, 2, 1, 3}, order)); + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {1025, 256, 1, 1}, order)); // test ceildiv in CUDA's CUB's DeviceSegmentedSort + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {2047, 2, 1, 3}, order)); + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {2048, 2, 1, 3}, order)); + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {2049, 2, 1, 3}, order)); + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {2, 8, 8192, 1}, order)); // bailingmoe2 (group selection) + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {2048, 512, 1, 1}, order)); // test CUDA dispatching to radix sort for nrows > = 1 in graph mode + } + + for (int n = 1; n < 5; ++n) { + for (int k = 1; k <= n; ++k) { + test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {n, 2, 1, 3}, k, true)); + } + } + for (int i = 0; i < 20; ++i) { + for (int k : {1, 2, 3, 7, 15, 100, 500, 1023, 9999}) { + if (k <= 1< 0.0f) continue; + for (float logit_softcap : {0.0f, 10.0f}) { + if (hsk != 128 && logit_softcap != 0.0f) continue; + for (int nh : { 1, 4 }) { + if (nh == 1 && hsk != 320 && hsk != 576) continue; + for (int nr3 : { 1, 3, }) { + if (hsk > 64 && nr3 > 1) continue; // skip broadcast for large head sizes + for (int nr2 : { 1, 4, 8, 12, 16, 20, 32 }) { + if (nr2 == 8 && hsk != 192) continue; + if (nr2 == 12 && hsk != 128) continue; + if (nr2 == 16 && hsk != 192) continue; + if (nr2 == 20 && (nh != 1 || hsk != 576)) continue; + if (nr2 == 32 && (nh != 1 || hsk != 320)) continue; + //for (int kv : { 1, 17, 31, 33, 61, 113, 65, 127, 129, 130, 255, 260, 371, 380, 407, 512, 1024, }) { + for (int kv : { 113, 512, 1024, }) { + if (nr2 != 1 && kv != 512) continue; + for (int nb : { 1, 3, 32, 75, }) { + for (ggml_prec prec : {GGML_PREC_F32, GGML_PREC_DEFAULT}) { + if (hsk != 128 && prec == GGML_PREC_DEFAULT) continue; + for (ggml_type type_KV : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL}) { + if (type_KV != GGML_TYPE_F16 && hsk != 64 && hsk != 72) continue; + test_cases.emplace_back(new test_flash_attn_ext( + hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV)); + // run fewer test cases permuted + if (mask == true && max_bias == 0.0f && logit_softcap == 0 && kv == 512) { + test_cases.emplace_back(new test_flash_attn_ext( + hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, type_KV, {0, 2, 1, 3})); + } + } + } + } + } + } + } + } + } + } + } + } + } + } + + // mixed quant and Q1_0 test cases + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(72, 72, 4, {1, 1}, 96, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 96, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F32)); + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {1, 1}, 256, 1, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0)); + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {1, 1}, 96, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q1_0, GGML_TYPE_Q1_0)); + test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q1_0, GGML_TYPE_Q4_0)); + test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q1_0)); + test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q1_0, GGML_TYPE_F16)); + + test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, { 10, 5, 4, 3})); + test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, {30000, 1, 1, 1})); + test_cases.emplace_back(new test_cross_entropy_loss_back(GGML_TYPE_F32, { 10, 5, 4, 3})); + test_cases.emplace_back(new test_cross_entropy_loss_back(GGML_TYPE_F32, {30000, 1, 1, 1})); + + test_cases.emplace_back(new test_opt_step_adamw(GGML_TYPE_F32, {10, 5, 4, 3})); + test_cases.emplace_back(new test_opt_step_sgd(GGML_TYPE_F32, {10, 5, 4, 3})); + + for (ggml_type type : base_types) { + for (bool with_gate : {false, true}) { + for (bool use_id : {false, true}) { + for (bool b : {false, true}) { + if (!use_id && b) { + continue; + } + for (bool with_bias : {false, true}) { + if (!with_gate && !with_bias) { + continue; + } + for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) { + if (!with_bias && glu_op == GGML_GLU_OP_SWIGLU_OAI) { + continue; + } + if (!with_gate && glu_op != GGML_GLU_OP_SWIGLU) { + continue; + } + for (bool with_lane_scale : {false, true}) { + if (with_lane_scale && type != GGML_TYPE_NVFP4) { + continue; + } + test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256, + use_id, 16, 8, b, with_bias, with_gate, with_lane_scale)); + test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256, + use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1})); + } + } + } + } + } + } + } + + for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT}) { + for (bool with_norm : {false, true}) { + for (bool bias_probs : {false, true}) { + for (float scale_w : {0.0f, 2.0f}) { + test_cases.emplace_back(new test_topk_moe({8, 22, 1, 1}, 4, with_norm, bias_probs, gate, scale_w)); + test_cases.emplace_back(new test_topk_moe({31, 22, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); + test_cases.emplace_back(new test_topk_moe({32, 22, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); + test_cases.emplace_back(new test_topk_moe({40, 22, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); + test_cases.emplace_back(new test_topk_moe({71, 22, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); + test_cases.emplace_back(new test_topk_moe({128, 1, 1, 1}, 128, with_norm, bias_probs, gate, scale_w)); + test_cases.emplace_back(new test_topk_moe({129, 1, 1, 1}, 128, with_norm, bias_probs, gate, scale_w)); + test_cases.emplace_back(new test_topk_moe({160, 4, 1, 1}, 160, with_norm, bias_probs, gate, scale_w)); + test_cases.emplace_back(new test_topk_moe({288, 22, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); // Used by StepFun 3.7 + } + } + } + } + + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 16, 1, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 16, 1, 1, 1, true, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 16, 1, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 64, 1, 2)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 2)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 8, 32, 4, 2, 2)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 2, 1, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 1, 1, true)); + // KDA (vector gate) + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 1, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 1, 2, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 16, 1, 2, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 32, 4, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 2, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 8, 32, 4, 2, 2, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 2, 1, true, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 16, 4, 2, 1, true, true)); + // chunked path: multi-chunk and non-multiple-of-chunk-size (chunk_size=64 GDN, 16 KDA) + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 64, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 127, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 256, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 65, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 100, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 200, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 127, 2)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 64, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 33, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 100, 1, 1, false, true)); + + // K > 1: output keeps the last min(n_tokens, K) per-token snapshots, ordered most-recent-first + // (slot 0 = final state, slot s = state s tokens back). + // exact-match cases (K == n_seq_tokens): + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 16, 2, 1, 1, false, false, /*K=*/2)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 32, 4, 1, 1, false, false, /*K=*/4)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 2, 1, false, false, /*K=*/4)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 8, 128, 4, 1, 1, false, false, /*K=*/4)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 2, 1, false, true, /*K=*/4)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 8, 32, 4, 2, 2, false, true, /*K=*/4)); + // overflow: n_tokens > K — only the last K snapshots kept. + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 32, 8, 1, 1, false, false, /*K=*/3)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 16, 2, 1, false, false, /*K=*/4)); + +#if 0 + // these tests are disabled to save execution time, sbut they can be handy for debugging + test_cases.emplace_back(new test_llama(2, true)); + test_cases.emplace_back(new test_llama(1)); + test_cases.emplace_back(new test_llama(2)); + test_cases.emplace_back(new test_falcon(1)); + test_cases.emplace_back(new test_falcon(2)); +#endif + + // lightning_indexer + for (int kv : { 256 }) { + for (int bs : { 1, 512 }) { + for (int nh : { 32, 64 }) { + for (auto [ns, nm] : { std::pair{1, 1}, std::pair{4, 4}, std::pair{4, 1} }) { + for (ggml_type type_K : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL}) { + test_cases.emplace_back(new test_lightning_indexer(128, nh, kv, bs, ns, nm, type_K)); + } + } + } + } + } + + return test_cases; +} +#ifdef _MSC_VER +#pragma optimize("", on) +#endif + +// Test cases for performance evaluation: should be representative of real-world use cases +static std::vector> make_test_cases_perf() { + std::vector> test_cases; + + // Conv2d: K=CRS=NPQ=4096 matmul performance + uint32_t iwh_idx = 0; + uint32_t kwh_idx = 1; + uint32_t Cout_idx = 2; + uint32_t Cin_idx = 3; + uint32_t B_idx = 4; + std::vector> cases = { + //{IWH, KWH, Cout, Cin, B} + // K=CRS=NPQ=4096 conv2d matmul performance + {19, 4, 4096, 256, 16}, + // K=128, CRS=128, NPQ=4096 + { 19, 4, 128, 8, 16}, + // K=130, CRS=128, NPQ=4096 + { 19, 4, 130, 8, 16}, + // Edge case: K x CRS is small + { 19, 2, 4, 4, 16}, + // A ConvNet's first layer + { 224, 3, 8, 3, 1 }, + // A ConvNet's first layer with 2x2 convolution, and 1 channel + { 224, 2, 8, 1, 1 }, + // A ConvNet's first layer with 2x2 convolution, and 1 channel, several images in the batch + { 224, 2, 8, 1, 8 }, + // A middle layer of a ConvNet + { 58, 3, 64, 32, 1 }, + // A middle layer of a ConvNet, several images in the batch + { 58, 3, 64, 32, 8 }, + // A deep layer of a ConvNet, several images in the batch + { 16, 3, 512, 128, 8 }, + // High resolution output (large NPQ) + {1536, 3, 64, 32, 1 }, + }; + + for (auto kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (auto act_case : cases) { + // Direct CONV_2D + test_cases.emplace_back(new test_conv_2d( + { act_case[iwh_idx], act_case[iwh_idx], act_case[Cin_idx], act_case[B_idx] }, + { act_case[kwh_idx], act_case[kwh_idx], act_case[Cin_idx], act_case[Cout_idx] }, + kernel_type, 1, 1, 0, 0, 1, 1, false)); + } + } + + struct conv3d_perf_case { + int N, IC, ID, IH, IW, OC, KD, KH, KW, s0, s1, s2, p0, p1, p2, d0, d1, d2; + }; + + const std::vector conv3d_cases = { + {1, 320, 8, 38, 26, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + {1, 1280, 8, 38, 26, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + {1, 320, 8, 76, 52, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + {1, 1280, 8, 76, 52, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + {1, 320, 8, 152, 104, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, +#if 0 + // too slow on some devices + {1, 1280, 8, 152, 104, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + {1, 320, 4, 304, 208, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + {1, 640, 4, 304, 208, 1280, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1}, +#endif + }; + + for (ggml_type kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (const conv3d_perf_case & c : conv3d_cases) { + test_cases.emplace_back(new test_conv_3d( + c.N, c.IC, c.ID, c.IH, c.IW, + c.OC, c.KD, c.KH, c.KW, + c.s0, c.s1, c.s2, c.p0, c.p1, c.p2, c.d0, c.d1, c.d2, + kernel_type)); + } + } + + test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {4096, 1, 1, 1}, {1, 1, 1, 1})); + test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {4096, 1, 1, 1}, {1, 512, 1, 1})); + + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F16, {512, 3072, 1, 1})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {8192, 512, 2, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {3072, 512, 2, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_Q4_0, {8192, 512, 2, 1})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_F32, {8192, 512, 2, 1})); + + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768*1024, 256, 1, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {0, 0, 0, 0})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768*1024, 256, 1, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {0, 0, 0, 0})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {0, 0, 0, 0})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {0, 0, 0, 0})); + + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768*1024, 256, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768*1024, 256, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); + test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true)); + + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {4096, 4096, 5, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {12888, 256, 5, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {77, 4096, 5, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {1024, 1024, 10, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {77, 1024, 10, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {256, 256, 20, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {64, 64, 20, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {77, 64, 20, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + + test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32, 10, 1, 1})); + test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {1024, 10, 1, 1})); + test_cases.emplace_back(new test_argmax(GGML_TYPE_F32, {32000, 512, 1, 1})); + + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {512, 34, 2, 1})); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 80, 1, 1})); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 80, 4, 1})); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 1, 1})); + test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 4, 1})); + + // SNAKE activation fusion at BigVGAN scale (T=7680 = 24 kHz x 320 ms, C=192) + test_cases.emplace_back(new test_snake_fuse(GGML_TYPE_F32, {7680, 192, 1, 1})); + test_cases.emplace_back(new test_snake_fuse(GGML_TYPE_F16, {7680, 192, 1, 1})); + test_cases.emplace_back(new test_snake_fuse(GGML_TYPE_BF16, {7680, 192, 1, 1})); + + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 16416, 1, 128, {8, 1}, {4, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 128, 1, 16416, {8, 1}, {4, 1}, {0, 1, 2, 3}, 2*16416)); + + // FWHT tests + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 1, 128)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 64, 1, 64)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 256, 1, 256)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 32, 128)); + + test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 64, 64, 4, 4 }, { 32, 64, 4, 4 })); + test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 128, 128, 4, 2 }, { 32, 128, 4, 2 })); + // qwen3next with CHUNK_SIZE 64 + test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 64, 64, 8, 32 }, { 64, 64, 8, 32 })); + // qwen3next with CHUNK_SIZE 128 + test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 128, 128, 4, 32 }, { 128, 128, 4, 32 })); + test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 256, 256, 4, 2 }, { 128, 256, 4, 2 })); + + test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER, GGML_TYPE_F32, { 256, 256, 4, 4 })); + test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER_DIAG, GGML_TYPE_F32, { 1024, 1024, 8, 4 })); + + test_cases.emplace_back(new test_cumsum(GGML_TYPE_F32, { 128, 128, 4, 4 })); + test_cases.emplace_back(new test_cumsum(GGML_TYPE_F32, { 2048, 16, 5, 4 })); + test_cases.emplace_back(new test_cumsum(GGML_TYPE_F32, { 20000, 10, 4, 1 })); + + for (int bs : {1, 2, 3, 4, 5, 8, 512}) { + for (ggml_type type_a : all_types) { + for (ggml_type type_b : {GGML_TYPE_F32}) { + test_cases.emplace_back(new test_mul_mat(type_a, type_b, 4096, bs, 14336, {1, 1}, {1, 1})); + } + } + } + + // qwen3-30b-a3b + for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { + for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) { + for (ggml_type type_b : {GGML_TYPE_F32}) { + test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, 128, 8, false, 768, bs, 2048)); + test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, type_b, 128, 8, false, 768, bs, 2048, 1)); + } + } + } + + for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { + for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) { + for (ggml_type type_b : {GGML_TYPE_F32}) { + test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, 32, 4, false, 1792, bs, 2048)); + test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, type_b, 32, 4, false, 1792, bs, 2048, 1)); + } + } + } + + + // gpt-oss-20b + for (int bs : {1, 4, 8, 512}) { + for (ggml_type type_a : {GGML_TYPE_MXFP4}) { + for (ggml_type type_b : {GGML_TYPE_F32}) { + test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, 32, 4, false, 2880, bs, 2880)); + test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, type_b, 32, 4, false, 2880, bs, 2880, 1)); + } + } + } + + for (int K : {3, 5}) { + for (int IC : {256, 2560}) { + for (int IW_IH : {32, 64, 256}) { + if (IC == 2560 && IW_IH == 256) { + // too big + continue; + } + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {IW_IH, IW_IH, IC, 1}, {K, K, IC, 1}, 1, 1, 1, 1, 1, 1, true)); + } + } + } + + // Qwen3-VL-8B https://github.com/ggml-org/llama.cpp/issues/17012 + test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 5776, 5776, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + + for (int kv : { 4096, 8192, 16384, }) { + for (int hs : { 64, 128, }) { + for (int nr : { 1, 4, }) { + test_cases.emplace_back(new test_flash_attn_ext(hs, hs, 8, {nr, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + } + } + } + + for (int col : {8192, 16384, 32768, 65536, 131072, 262144, 524288}) { + for (int rows : {1, 4, 16}){ + test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {col, rows, 1, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f)); + } + } + + test_cases.emplace_back(new test_conv_2d_dw({512, 512, 256, 1}, {3, 3, 1, 256}, GGML_TYPE_F32, 1, 1, 1, false)); + test_cases.emplace_back(new test_conv_2d_dw({512, 512, 256, 1}, {3, 3, 1, 256}, GGML_TYPE_F32, 1, 1, 1, true)); + test_cases.emplace_back(new test_conv_2d_dw({112, 112, 32, 1}, {3, 3, 1, 32}, GGML_TYPE_F32, 1, 1, 1, false)); + test_cases.emplace_back(new test_conv_2d_dw({112, 112, 32, 1}, {3, 3, 1, 32}, GGML_TYPE_F32, 1, 1, 1, true)); + test_cases.emplace_back(new test_conv_2d_dw({56, 56, 128, 1}, {5, 5, 1, 128}, GGML_TYPE_F32, 2, 2, 1, false)); + test_cases.emplace_back(new test_conv_2d_dw({56, 56, 128, 1}, {5, 5, 1, 128}, GGML_TYPE_F32, 2, 2, 1, true)); + + for (ggml_type kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + test_cases.emplace_back(new test_conv_transpose_2d({256, 256, 256, 1}, {3, 3, 16, 256}, 1, kernel_type)); + test_cases.emplace_back(new test_conv_transpose_2d({16, 16, 16, 1}, {3, 3, 8, 16}, 1, kernel_type)); + test_cases.emplace_back(new test_conv_transpose_2d({10, 10, 9, 1}, {3, 3, 1, 9}, 2, kernel_type)); + } + + // Memory bound overlap-add of the GEMM + col2im_1d transposed conv path, real vocoder stage shapes + test_cases.emplace_back(new test_col2im_1d(GGML_TYPE_F32, 16, 512, 2048, 8, 0)); + test_cases.emplace_back(new test_col2im_1d(GGML_TYPE_F32, 4, 128, 65536, 2, 0)); + test_cases.emplace_back(new test_col2im_1d(GGML_TYPE_F16, 16, 512, 2048, 8, 0)); + + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, {256, 256, 3, 1})); + + + for (int n_token : {1, 512}) { + test_cases.emplace_back(new test_add_id(GGML_TYPE_F32, GGML_TYPE_F32, 2880, 128, 4, n_token)); + test_cases.emplace_back(new test_add_id(GGML_TYPE_F32, GGML_TYPE_F32, 2880, 32, 4, n_token)); + } + + for (bool fw : {true, false}) { // fw == forward + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (bool ff : {false, true}) { // freq_factors + for (float v : { 0, 1 }) { + test_cases.emplace_back(new test_rope(type, {128, 32, 512, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // llama 7B + test_cases.emplace_back(new test_rope(type, {128, 64, 512, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // llama 65B + test_cases.emplace_back(new test_rope(type, { 80, 32, 512, 1}, 20, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // neox (stablelm) + test_cases.emplace_back(new test_rope(type, { 64, 8, 512, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // neox (falcon 40B) + test_cases.emplace_back(new test_rope(type, {128, 12, 512, 1}, 128, GGML_ROPE_TYPE_MROPE, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // rope_multi,m-rope (qwen2vl 2B) + test_cases.emplace_back(new test_rope(type, {128, 12, 512, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // rope_multi,imrope (qwen3vl 2B) + test_cases.emplace_back(new test_rope(type, { 80, 16, 2, 1}, 80, GGML_ROPE_TYPE_VISION, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // rope_multi,m-rope (qwen2vl ViT) + } + } + } + } + + std::vector> reduce_rows_cases = { + { 8192, 1, 1, 1 }, + { 8192, 8192, 1, 1 }, + { 128, 8192, 1, 1 }, + }; + + for (auto it: reduce_rows_cases){ + test_cases.emplace_back(new test_mean(GGML_TYPE_F32, it)); + test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, it)); + test_cases.emplace_back(new test_sum(GGML_TYPE_F32, it)); + } + + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {65000, 16, 1, 1})); + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {200000, 1, 1, 1})); + test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {200000, 16, 1, 1})); + + test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {2, 1, 1, 1}, 1)); + for (auto k : {1, 10, 40, 400}) { + for (auto nrows : {1, 16}) { + for (auto cols : {k, 1000, 65000, 200000}) { + test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {cols, nrows, 1, 1}, k)); + } + } + } + + for (auto nrows : {1, 4, 8, 16}) { + for (auto cols : {128, 1024, 4096, 8192, 16384, 32768, 65536, 131072, 200000, 2000000}) { + test_cases.emplace_back(new test_cumsum(GGML_TYPE_F32, {cols, nrows, 1, 1})); + } + } + + // Examples from granite-4.0-h-1b/ggml-model-Q8_0.gguf + test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {515, 3328, 1, 1}, {4, 3328, 1, 1})); // prefill + test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {937, 8192, 1, 1}, {4, 8192, 1, 1})); // prefill + test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {4, 3328, 1, 1}, {4, 3328, 1, 1})); // generate + test_cases.emplace_back(new test_ssm_conv_bias_silu(GGML_TYPE_F32, {515, 3328, 1, 1}, {4, 3328, 1, 1}, true)); // prefill + test_cases.emplace_back(new test_ssm_conv_bias_silu(GGML_TYPE_F32, {4, 3328, 1, 1}, {4, 3328, 1, 1}, true)); // generate + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 48, 1, 512, 1)); // prefill + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 48, 1, 1, 1)); // generate + + // acc + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 1, 1}, {256, 16, 1, 1}, -1)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {256, 16, 2, 3}, -1)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {128, 16, 2, 3}, -1)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {256, 16, 2, 3}, 1)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {128, 16, 2, 3}, 2)); + test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {64, 16, 2, 3}, 3)); + + // GATED_DELTA_NET: realistic model configurations + // TG: n_seq_tokens=1 (autoregressive) + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1)); // Qwen3.5-like: 32 heads, d=128 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 64, 1, 1)); // smaller model + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1, 1, false, true)); // KDA + // PP: n_seq_tokens=64,256 (prompt processing) + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 64, 1)); // PP-64 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 256, 1)); // PP-256 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 512, 1)); // PP-512 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1024, 1)); // PP-1024 + // Small model configs (fewer heads = less GPU occupancy for autoregressive) + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1)); // 4h PP-64 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 256, 1)); // 4h PP-256 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 512, 1)); // 4h PP-512 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1024, 1)); // 4h PP-1024 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 64, 1, 1, false, true)); // KDA PP-64 + + // lightning_indexer + for (int kv : { 256, 4096, 65536 }) { + for (int bs : { 1, 512, 2048 }) { + for (int nh : { 32, 64 }) { + for (int ns : { 1, 4 }) { + for (ggml_type type_K : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL}) { + test_cases.emplace_back(new test_lightning_indexer(128, nh, kv, bs, ns, ns, type_K)); + } + } + } + } + } + + return test_cases; +} + +static std::vector> make_test_cases_from_file(const char * path) { + std::ifstream f(path); + + if (!f.is_open()) { + throw std::runtime_error("Unable to read test file"); + } + + std::vector> test_cases; + + std::string line; + + while (std::getline(f, line)) { + std::istringstream iss(line); + + ggml_op op; + ggml_type type; + std::array ne; + std::array op_params = {}; + std::string name; + uint64_t tmp; + + iss >> tmp; + op = (ggml_op)tmp; + iss >> tmp; + type = (ggml_type)tmp; + + for (size_t i = 0; i < 4; i++) { + iss >> ne[i]; + } + + iss >> tmp; + for (size_t i = 0; i < tmp && i < op_params.size(); i++) { + iss >> op_params[i]; + } + + iss >> tmp; + + size_t num_src = std::min((uint64_t)GGML_MAX_SRC, tmp); + std::vector sources(num_src); + for (size_t i = 0; i < num_src; i++) { + input_tensor& src = sources[i]; + iss >> tmp; + src.type = (ggml_type)tmp; + + for (size_t i = 0; i < 4; i++) { + iss >> src.ne[i]; + } + for (size_t i = 0; i < 4; i++) { + iss >> src.nb[i]; + } + } + + iss >> name; + + if (name.length() == 1 && name[0] == '-') { + name = ""; + } + + test_cases.emplace_back(new test_generic_op(op, type, ne, op_params, sources, std::move(name))); + } + + return test_cases; +} + +static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mode mode, const char * op_names_filter, const char * params_filter, + printer * output_printer, const char * test_file_path, int parallel_workers) { + auto filter_test_cases = [](std::vector> & test_cases, const char * params_filter) { + if (params_filter == nullptr) { + return; + } + + std::regex params_filter_regex(params_filter); + + for (auto it = test_cases.begin(); it != test_cases.end();) { + if (!std::regex_search((*it)->vars(), params_filter_regex)) { + it = test_cases.erase(it); + continue; + } + + it++; + } + }; + + std::vector> test_cases; + + if (test_file_path == nullptr) { + switch (mode) { + case MODE_TEST: + case MODE_GRAD: + case MODE_SUPPORT: + test_cases = make_test_cases_eval(); + break; + case MODE_PERF: + test_cases = make_test_cases_perf(); + break; + } + } else { + test_cases = make_test_cases_from_file(test_file_path); + } + + filter_test_cases(test_cases, params_filter); + + if (mode == MODE_TEST) { + ggml_backend_ptr backend_cpu(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL)); + if (backend_cpu == NULL) { + test_operation_info info("", "", "CPU"); + info.set_error("backend", "Failed to initialize CPU backend"); + output_printer->print_operation(info); + return false; + } + // Use reference implementation on the CPU backend for comparison + using ggml_backend_cpu_set_use_ref_t = void (*)(ggml_backend_t, bool); + auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu.get())); + auto * set_use_ref = (ggml_backend_cpu_set_use_ref_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_use_ref"); + if (set_use_ref) { + set_use_ref(backend_cpu.get(), true); + } + + std::atomic n_ok = 0; + std::atomic tests_run = 0; + std::vector failed_tests; + std::mutex failed_tests_mutex; + + // Each worker grabs a chunk of cases at a time. The chunk shrinks as we + // run out of work so that a few slow tests at the tail get spread across + // workers instead of landing on one unlucky thread. + constexpr size_t MAX_TESTS_PER_ITER = 100; + std::atomic test_idx = 0; + + const auto & next_chunk = [&](size_t & my_begin, size_t & my_end) { + const size_t cur = test_idx.load(std::memory_order_relaxed); + const size_t remaining = cur < test_cases.size() ? test_cases.size() - cur : 0; + const size_t chunk = std::max(1, std::min(MAX_TESTS_PER_ITER, remaining / parallel_workers)); + my_begin = test_idx.fetch_add(chunk); + my_end = std::min(my_begin + chunk, test_cases.size()); + }; + + const auto & run_tests = [&](ggml_backend_t b, ggml_backend_t b_cpu) { + size_t my_begin, my_end; + next_chunk(my_begin, my_end); + while (my_begin < test_cases.size()) { + for (size_t i = my_begin; i < my_end; ++i) { + auto & test = test_cases[i]; + test_status_t status = test->eval(b, b_cpu, op_names_filter, output_printer); + if (status == test_status_t::SKIPPED || status == test_status_t::NOT_SUPPORTED) { + continue; + } + tests_run++; + if (status == test_status_t::OK) { + n_ok++; + } else if (status == test_status_t::FAIL) { + std::lock_guard guard(failed_tests_mutex); + failed_tests.push_back(test->current_op_name + "(" + test->vars() + ")"); + } + } + next_chunk(my_begin, my_end); + } + }; + + if (parallel_workers <= 1) { + // Reuse the outer backend / backend_cpu so we don't pay an + // extra CPU backend init. + run_tests(backend, backend_cpu.get()); + } else { + std::atomic workers_started = 0; + + const auto & eval_worker = [&]() { + ggml_backend_ptr b(ggml_backend_dev_init(dev, NULL)); + if (b == NULL) { + return; + } + + ggml_backend_ptr b_cpu(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL)); + if (b_cpu == NULL) { + return; + } + + if (set_use_ref) { + set_use_ref(b_cpu.get(), true); + } + workers_started++; + run_tests(b.get(), b_cpu.get()); + }; + + std::vector threads; + threads.reserve(parallel_workers); + for (int i = 0; i < parallel_workers; ++i) { + threads.emplace_back(eval_worker); + } + for (auto & t : threads) { + t.join(); + } + + if (workers_started == 0 && !test_cases.empty()) { + return false; + } + } + + output_printer->print_summary(test_summary_info(n_ok, tests_run, false)); + output_printer->print_failed_tests(failed_tests); + + return n_ok == tests_run; + } + + if (mode == MODE_GRAD) { + test_cases.erase( + std::remove_if(test_cases.begin(), test_cases.end(), [](const std::unique_ptr & tc) { + return tc->run_whole_graph(); + }), + test_cases.end() + ); + + size_t n_ok = 0; + for (auto & test : test_cases) { + if (test->eval_grad(backend, op_names_filter, output_printer)) { + n_ok++; + } + } + output_printer->print_summary(test_summary_info(n_ok, test_cases.size(), false)); + + return n_ok == test_cases.size(); + } + + if (mode == MODE_PERF) { + for (auto & test : test_cases) { + test->eval_perf(backend, op_names_filter, output_printer); + } + return true; + } + + if (mode == MODE_SUPPORT) { + // Filter out fusion cases + test_cases.erase( + std::remove_if(test_cases.begin(), test_cases.end(), [](const std::unique_ptr & tc) { + return tc->run_whole_graph(); + }), + test_cases.end() + ); + + for (auto & test : test_cases) { + test->eval_support(backend, op_names_filter, output_printer); + } + return true; + } + + GGML_ABORT("fatal error"); +} + +static void list_all_ops() { + printf("GGML operations:\n"); + std::set all_ops; + + for (int i = 1; i < GGML_OP_COUNT; i++) { + all_ops.insert(ggml_op_name((enum ggml_op)i)); + } + for (int i = 0; i < GGML_UNARY_OP_COUNT; i++) { + all_ops.insert(ggml_unary_op_name((enum ggml_unary_op)i)); + } + for (int i = 0; i < GGML_GLU_OP_COUNT; i++) { + all_ops.insert(ggml_glu_op_name((enum ggml_glu_op)i)); + } + for (const auto & op : all_ops) { + printf(" %s\n", op.c_str()); + } + printf("\nTotal: %zu operations\n", all_ops.size()); +} + +static void show_test_coverage() { + std::set all_ops; + for (int i = 1; i < GGML_OP_COUNT; i++) { + auto op = (enum ggml_op)i; + if (op == GGML_OP_VIEW || + op == GGML_OP_RESHAPE || + op == GGML_OP_PERMUTE || + op == GGML_OP_TRANSPOSE || + op == GGML_OP_CONT || + op == GGML_OP_GLU || + op == GGML_OP_UNARY) { + continue; + } + all_ops.insert(ggml_op_name(op)); + } + for (int i = 0; i < GGML_UNARY_OP_COUNT; i++) { + all_ops.insert(ggml_unary_op_name((enum ggml_unary_op)i)); + } + for (int i = 0; i < GGML_GLU_OP_COUNT; i++) { + all_ops.insert(ggml_glu_op_name((enum ggml_glu_op)i)); + } + auto test_cases = make_test_cases_eval(); + // Filter out fusion cases + test_cases.erase( + std::remove_if(test_cases.begin(), test_cases.end(), [](const std::unique_ptr & tc) { + return tc->run_whole_graph(); + }), + test_cases.end() + ); + + std::set tested_ops; + + ggml_init_params params = { + /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead(), + /* .mem_base = */ NULL, + /* .no_alloc = */ true, + }; + + for (auto & test_case : test_cases) { + ggml_context_ptr ctx(ggml_init(params)); + if (ctx) { + test_case->mode = MODE_TEST; + ggml_tensor * out = test_case->build_graph(ctx.get()); + if (out && out->op != GGML_OP_NONE) { + if (out->op == GGML_OP_UNARY) { + tested_ops.insert(ggml_unary_op_name(ggml_get_unary_op(out))); + } else if (out->op == GGML_OP_GLU) { + tested_ops.insert(ggml_glu_op_name(ggml_get_glu_op(out))); + } else { + tested_ops.insert(ggml_op_name(out->op)); + } + } + } + } + std::set covered_ops; + std::set uncovered_ops; + for (const auto & op : all_ops) { + if (tested_ops.count(op) > 0) { + covered_ops.insert(op); + } else { + uncovered_ops.insert(op); + } + } + + printf("Operations covered by tests (%zu):\n", covered_ops.size()); + for (const auto & op : covered_ops) { + printf(" ✓ %s\n", op.c_str()); + } + printf("\nOperations without tests (%zu):\n", uncovered_ops.size()); + for (const auto & op : uncovered_ops) { + printf(" ✗ %s\n", op.c_str()); + } + + printf("\nCoverage Summary:\n"); + printf(" Total operations: %zu\n", all_ops.size()); + printf(" Tested operations: %zu\n", covered_ops.size()); + printf(" Untested operations: %zu\n", uncovered_ops.size()); + printf(" Coverage: %.1f%%\n", (double)covered_ops.size() / all_ops.size() * 100.0); +} + +static void usage(char ** argv) { + printf("Usage: %s [mode] [-o ] [-b ] [-p ] [--output ] [--list-ops]", argv[0]); + printf(" [--show-coverage] [--test-file ] [-j ]\n"); + printf(" valid modes:\n"); + printf(" - test (default, compare with CPU backend for correctness)\n"); + printf(" - grad (compare gradients from backpropagation with method of finite differences)\n"); + printf(" - perf (performance evaluation)\n"); + printf(" - support (probe backend operation support)\n"); + printf(" op names for -o are as given by ggml_op_desc() (e.g. ADD, MUL_MAT, etc),\n"); + printf(" optionally including the full test case string (e.g. \"ADD(type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1)\")\n"); + printf(" --output specifies output format (default: console, options: console, sql, csv)\n"); + printf(" --list-ops lists all available GGML operations\n"); + printf(" --show-coverage shows test coverage\n"); + printf(" --test-file reads test operators from a test file generated by test-export-graph-ops\n"); + printf(" -j runs tests using parallel worker threads (default: 1, test mode only)\n"); +} + +int main(int argc, char ** argv) { + test_mode mode = MODE_TEST; + output_formats output_format = CONSOLE; + const char * op_names_filter = nullptr; + const char * backend_filter = nullptr; + const char * params_filter = nullptr; + const char * test_file_path = nullptr; + int parallel_workers = 1; + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "test") == 0) { + mode = MODE_TEST; + } else if (strcmp(argv[i], "perf") == 0) { + mode = MODE_PERF; + } else if (strcmp(argv[i], "grad") == 0) { + mode = MODE_GRAD; + } else if (strcmp(argv[i], "support") == 0) { + mode = MODE_SUPPORT; + } else if (strcmp(argv[i], "-o") == 0) { + if (i + 1 < argc) { + op_names_filter = argv[++i]; + } else { + usage(argv); + return 1; + } + } else if (strcmp(argv[i], "-b") == 0) { + if (i + 1 < argc) { + backend_filter = argv[++i]; + } else { + usage(argv); + return 1; + } + } else if (strcmp(argv[i], "-p") == 0) { + if (i + 1 < argc) { + params_filter = argv[++i]; + } else { + usage(argv); + return 1; + } + } else if (strcmp(argv[i], "--output") == 0) { + if (i + 1 < argc) { + if (!output_format_from_str(argv[++i], output_format)) { + usage(argv); + return 1; + } + } else { + usage(argv); + return 1; + } + } else if (strcmp(argv[i], "--list-ops") == 0) { + list_all_ops(); + return 0; + } else if (strcmp(argv[i], "--show-coverage") == 0) { + show_test_coverage(); + return 0; + } else if (strcmp(argv[i], "--test-file") == 0) { + if (i + 1 < argc) { + test_file_path = argv[++i]; + } else { + usage(argv); + return 1; + } + } else if (strcmp(argv[i], "-j") == 0) { + if (i + 1 < argc) { + parallel_workers = atoi(argv[++i]); + if (parallel_workers < 1) { + usage(argv); + return 1; + } + } else { + usage(argv); + return 1; + } + } else { + usage(argv); + return 1; + } + } + + // load and enumerate backends + ggml_backend_load_all(); + + // Create printer for output format + std::unique_ptr output_printer = create_printer(output_format); + if (output_printer) { + output_printer->print_header(); + } + + output_printer->print_testing_start(testing_start_info(ggml_backend_dev_count())); + + size_t n_ok = 0; + + for (size_t i = 0; i < ggml_backend_dev_count(); i++) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + + if (backend_filter != NULL && strcmp(backend_filter, ggml_backend_dev_name(dev)) != 0) { + output_printer->print_backend_init( + backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping")); + n_ok++; + continue; + } + + if (backend_filter == NULL && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU && mode != MODE_GRAD) { + output_printer->print_backend_init(backend_init_info( + i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping CPU backend")); + n_ok++; + continue; + } + + ggml_backend_ptr backend(ggml_backend_dev_init(dev, NULL)); + GGML_ASSERT(backend != NULL); + + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); + auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); + if (ggml_backend_set_n_threads_fn) { + // TODO: better value for n_threads + ggml_backend_set_n_threads_fn(backend.get(), N_THREADS); + } + + size_t free, total; // NOLINT + ggml_backend_dev_memory(dev, &free, &total); + output_printer->print_backend_init(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), + false, "", ggml_backend_dev_description(dev), + total / 1024 / 1024, free / 1024 / 1024, true)); + + bool ok = test_backend(backend.get(), dev, mode, op_names_filter, params_filter, output_printer.get(), test_file_path, parallel_workers); + + if (ok) { + n_ok++; + } + output_printer->print_backend_status( + backend_status_info(ggml_backend_name(backend.get()), ok ? test_status_t::OK : test_status_t::FAIL)); + } + + ggml_quantize_free(); + + if (output_printer) { + output_printer->print_footer(); + } + + output_printer->print_overall_summary( + overall_summary_info(n_ok, ggml_backend_dev_count(), n_ok == ggml_backend_dev_count())); + + if (n_ok != ggml_backend_dev_count()) { + return 1; + } + + return 0; +} From 279ecd8fa6839a068f79f633d58e5d1e20b9acda Mon Sep 17 00:00:00 2001 From: marty1885 Date: Tue, 14 Jul 2026 14:03:38 +0000 Subject: [PATCH 09/12] repair build --- ggml/src/ggml-et/et-kernels/src/platform.h | 8 -------- ggml/src/ggml-et/ggml-et-cpu-compare.cpp | 11 ----------- 2 files changed, 19 deletions(-) diff --git a/ggml/src/ggml-et/et-kernels/src/platform.h b/ggml/src/ggml-et/et-kernels/src/platform.h index 51af928580d9..44854052fd64 100644 --- a/ggml/src/ggml-et/et-kernels/src/platform.h +++ b/ggml/src/ggml-et/et-kernels/src/platform.h @@ -142,14 +142,6 @@ static inline void atomic_store_f32(volatile float * addr, float value) { et_global_swap_w(addr, *(uint32_t *) &value); } -static inline void atomic_add_f32(volatile float * addr, float value) { - et_global_add_w(addr, *(uint32_t *) &value); -} - -static inline void atomic_store_f16(volatile uint16_t * addr, uint16_t value) { - et_global_store_hw(addr, value); -} - // Atomic add for F32 values to global memory // Uses ET hardware's custom amoaddg.w instruction for global atomic add // This ensures correct accumulation when multiple threads contribute to the same output diff --git a/ggml/src/ggml-et/ggml-et-cpu-compare.cpp b/ggml/src/ggml-et/ggml-et-cpu-compare.cpp index ad9fd9f127e4..12ed94262a31 100644 --- a/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +++ b/ggml/src/ggml-et/ggml-et-cpu-compare.cpp @@ -286,9 +286,6 @@ bool ggml_et_cpu_compare_compute_and_check(ggml_et_cpu_compare_ctx * ct } } break; - case GGML_OP_GET_ROWS: - ctx->cpu_dst = ggml_get_rows(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1); - break; case GGML_OP_CONT: ctx->cpu_dst = ggml_cont(ctx->ggml_ctx, ctx->cpu_src0); break; @@ -311,14 +308,6 @@ bool ggml_et_cpu_compare_compute_and_check(ggml_et_cpu_compare_ctx * ct ctx->cpu_dst = ggml_set_rows(ctx->ggml_ctx, cpu_dst_base, ctx->cpu_src0, ctx->cpu_src1); } break; - case GGML_OP_RMS_NORM: - // Extract epsilon parameter from op_params (stored as float) - { - float eps; - memcpy(&eps, node->op_params, sizeof(float)); - ctx->cpu_dst = ggml_rms_norm(ctx->ggml_ctx, ctx->cpu_src0, eps); - } - break; case GGML_OP_GET_ROWS: ctx->cpu_dst = ggml_get_rows(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1); break; From b01e83d8961ee87245f2f12831bebe8611a6b0b3 Mon Sep 17 00:00:00 2001 From: Saqib Akram Date: Wed, 15 Jul 2026 21:53:52 +0500 Subject: [PATCH 10/12] ET backend: Q4_K MUL_MAT (scalar + matrix engine) + uberkernel perf recovery (#15) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds Q4_K MUL_MAT to the ET (ETSOC-1) backend — a scalar kernel plus a matrix-engine kernel for prefill — improves the existing Q4_0/F16/F32 matrix-engine kernels, and fixes a Q8_0 generation regression in the uberkernel path. Verified on Llama-3.2-1B-Instruct on ETSOC-1 with flash attention enabled. --- ggml/src/ggml-et/CMakeLists.txt | 3 + ggml/src/ggml-et/et-kernels/src/block_ops.h | 72 +++ .../et-kernels/src/flash_attn_ext_f16_me.c | 36 +- .../src/mul_mat_Q4_0_matrix_engine.c | 537 ++++++++++------ .../src/ggml-et/et-kernels/src/mul_mat_Q4_K.c | 326 ++++++++++ .../src/mul_mat_Q4_K_matrix_engine.c | 595 ++++++++++++++++++ .../src/mul_mat_f16_matrix_engine.c | 211 ++++--- .../src/mul_mat_f32_matrix_engine.c | 114 ++-- ggml/src/ggml-et/et-kernels/src/platform.h | 28 +- ggml/src/ggml-et/ggml-et-ops.cpp | 14 +- ggml/src/ggml-et/ggml-et.cpp | 21 + tests/test-backend-ops.cpp | 9 + 12 files changed, 1633 insertions(+), 333 deletions(-) create mode 100644 ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K.c create mode 100644 ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c diff --git a/ggml/src/ggml-et/CMakeLists.txt b/ggml/src/ggml-et/CMakeLists.txt index ee0ee3759a91..1c89e536257d 100644 --- a/ggml/src/ggml-et/CMakeLists.txt +++ b/ggml/src/ggml-et/CMakeLists.txt @@ -38,6 +38,8 @@ set(KERNELS mul_mat_Q8_0 mul_mat_Q4_0 mul_mat_Q4_0_matrix_engine + mul_mat_Q4_K + mul_mat_Q4_K_matrix_engine mul_mat_f16 mul_mat_f16_matrix_engine rope_f32 @@ -103,6 +105,7 @@ set(UBERKERNEL_SUPPORTED_KERNELS mul_mat_f32_matrix_engine mul_mat_id_f32 mul_mat_Q4_0 + mul_mat_Q4_K mul_mat_Q8_0 norm_f32 pad_f32 diff --git a/ggml/src/ggml-et/et-kernels/src/block_ops.h b/ggml/src/ggml-et/et-kernels/src/block_ops.h index 78ffbde87bfa..5db35244302a 100644 --- a/ggml/src/ggml-et/et-kernels/src/block_ops.h +++ b/ggml/src/ggml-et/et-kernels/src/block_ops.h @@ -882,6 +882,7 @@ static inline void q4_dot_compute_x2_aligned(const block_q4_0 * q_row0, float * out1) { const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; __asm__ volatile("flw.ps f31, %[g]\n" : : [g] "m"(*(const int32_t (*)[8]) gather_pattern) : "f31"); + __asm__ volatile( "fbci.pi f20, 0\n" "fbci.pi f21, 0\n" :: @@ -995,3 +996,74 @@ static inline void q4_dot_compute_x2_aligned(const block_q4_0 * q_row0, *out0 = result0; *out1 = result1; } + + +// Full-row dot product for Q4_K weights against an F32 activation column. +// +// Unlike Q4_0/Q8_0 (whose dequant is a pure per-block scale, so the scale can +// be factored out of the dot product), Q4_K reconstructs each weight via an +// affine transform `w = d*scale*q - dmin*min` with per-group scales/mins inside +// each 256-element super-block. That makes the cheap "scale the integer dot" +// trick inapplicable. +// +// The dequant math mirrors dequantize_q4_K_block exactly, but the per-element +// product is folded straight into a scalar accumulator instead of being staged +// through a temporary buffer. This deliberately avoids a large (1KB) on-stack +// dequant buffer and the vector-mask save/restore of the F32 dot helper, both +// of which are unsafe in the uberkernel context. +// +// K_sblocks is the number of QK_K (256) element super-blocks in the row +// (i.e. K / QK_K). +static inline float sw_fp16_to_fp32(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000) << 16; + uint32_t exp = (h >> 10) & 0x1F; + uint32_t mant = h & 0x3FF; + uint32_t f; + if (exp == 0) { + if (mant == 0) { f = sign; } + else { + exp = 127 - 15 + 1; + while ((mant & 0x400) == 0) { mant <<= 1; exp--; } + mant &= 0x3FF; + f = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1F) { + f = sign | 0x7F800000u | (mant << 13); + } else { + f = sign | ((exp + (127 - 15)) << 23) | (mant << 13); + } + float out; __builtin_memcpy(&out, &f, 4); return out; +} + +static inline float compute_row_dot_q4_K(const block_q4_K* q_row, + const float* b_col, + int64_t K_sblocks) { + float acc = 0.0f; + for (int64_t sb = 0; sb < K_sblocks; sb++) { + const block_q4_K* block = q_row + sb; + const float* b = b_col + sb * QK_K; + const uint8_t* q = block->qs; + const float d = sw_fp16_to_fp32(block->d); + const float min = sw_fp16_to_fp32(block->dmin); + + int is = 0; + uint8_t sc, m; + for (int j = 0; j < QK_K; j += 64) { + get_scale_min_k4(is + 0, block->scales, &sc, &m); + const float d1 = d * sc; + const float m1 = min * m; + get_scale_min_k4(is + 1, block->scales, &sc, &m); + const float d2 = d * sc; + const float m2 = min * m; + for (int l = 0; l < 32; ++l) { + acc += (d1 * (float)(q[l] & 0xF) - m1) * (*b++); + } + for (int l = 0; l < 32; ++l) { + acc += (d2 * (float)(q[l] >> 4) - m2) * (*b++); + } + q += 32; + is += 2; + } + } + return acc; +} diff --git a/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c b/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c index c905b366f380..7c93ad778518 100644 --- a/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +++ b/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c @@ -358,9 +358,12 @@ int entry_point(struct ggml_et_flash_attn_ext_params * params, void * env) { const int is_hart1 = hart_id & 1; uint64_t local_minion = (hart_id >> 1) & 0x1F; - struct ggml_tensor * q = ¶ms->src0; - struct ggml_tensor * k = ¶ms->src1; - struct ggml_tensor * v = ¶ms->src2; + struct ggml_tensor * q = ¶ms->src0; + evict_region_past_l2(q->data, tensor_bytes_fa(q)); + struct ggml_tensor * k = ¶ms->src1; + evict_region_past_l2(k->data, tensor_bytes_fa(k)); + struct ggml_tensor * v = ¶ms->src2; + evict_region_past_l2(v->data, tensor_bytes_fa(v)); struct ggml_tensor * dst = ¶ms->dst; const int32_t has_mask = params->has_mask; struct ggml_tensor * mask = has_mask ? ¶ms->mask : (struct ggml_tensor *) 0; @@ -371,13 +374,13 @@ int entry_point(struct ggml_et_flash_attn_ext_params * params, void * env) { char * dst_data = (char *) dst->data; // et_barrier(ET_BARRIER_GLOBAL); - evict_region_past_l2(q->data, tensor_bytes_fa(q)); - evict_region_past_l2(k->data, tensor_bytes_fa(k)); - evict_region_past_l2(v->data, tensor_bytes_fa(v)); - if (mask) { - evict_region_past_l2(mask->data, tensor_bytes_fa(mask)); - } - et_barrier(ET_BARRIER_GLOBAL); + // evict_region_past_l2(q_data, tensor_bytes_fa(q)); + // evict_region_past_l2(k_data, tensor_bytes_fa(k)); + // evict_region_past_l2(v_data, tensor_bytes_fa(v)); + // if (mask) { + // evict_region_past_l2(mask->data, tensor_bytes_fa(mask)); + // } + // et_barrier(ET_BARRIER_GLOBAL); const int64_t dk = q->ne[0]; const int64_t nq = q->ne[1]; @@ -462,9 +465,14 @@ int entry_point(struct ggml_et_flash_attn_ext_params * params, void * env) { // All teams in a shire must iterate the same number of times so the // per-iter shire barriers stay balanced. Teams whose assigned row is // past total_rows still call the barriers but skip the packing work. + et_barrier(ET_BARRIER_SHIRE); // et_barrier(ET_BARRIER_GLOBAL); + if (is_hart1) { + // et_barrier(ET_BARRIER_GLOBAL); + // et_barrier(ET_BARRIER_SHIRE); + uint32_t chunk_id = 0; const int64_t row_base = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; @@ -548,10 +556,10 @@ int entry_point(struct ggml_et_flash_attn_ext_params * params, void * env) { } // Hart 0: tensor engine compute -#ifndef UBERKERNEL_SUPPRESS_SCP_SETUP - setup_cache_scp(); -#endif - CLEAR_TENSOR_ERROR; + // #ifndef UBERKERNEL_SUPPRESS_SCP_SETUP + // setup_cache_scp(); + // #endif + // CLEAR_TENSOR_ERROR; // Q converted to F16 (one row at a time) et_fp16_t q_f16[FA_DK_MAX] __attribute__((aligned(64))); diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c index 28a10303235a..c3af3addd4a1 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c @@ -1,54 +1,85 @@ +#include +#include #include "ggml_tensor.h" -#include "math_fp.h" #include "platform.h" -#include "quants.h" #include "tensor.h" - -#include -#include +#include "quants.h" +#include "math_fp.h" // Q4_0 x F32 -> F32 MUL_MAT on the tensor (matrix) engine, TensorFMA32. // Hart 1: dequantize Q4_0 weights to FP32 into double-buffered L2 SCP. // Hart 0: tensor engine compute (FMA, reduce, store). +// +// Two execution paths (selected at runtime by N % TILE_N): +// * REUSE path (N % TILE_N == 0): dequantize each weight K-window ONCE and +// reuse it across ru_n consecutive N-tiles, so the (producer-bound) +// dequant work is cut by ~ru_n. Partial C is round-tripped through an +// L2-SCP scratch between K-windows (the FMA C accumulator is a single fixed +// register-file tile, so multiple output tiles cannot be resident at once). +// * ORIGINAL path (N % TILE_N != 0): one output tile at a time, no reuse. #define NUM_COMPUTE_SHIRES 32 #define MINIONS_PER_SHIRE 32 #define TILE_M 16 #define TILE_N 16 -#define BLOCK_K QK4_0 // 32 elements per Q4_0 block -#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) +#define BLOCK_K QK4_0 // 32 elements per Q4_0 block +#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) + +// --- Reuse knobs ---------------------------------------------------------- +// REUSE_MAX caps the L2-SCP C-scratch footprint; the actual reuse factor is +// chosen at runtime (see ru_n) as the largest value that still keeps the whole +// machine busy. KWIN is the dequant-cache depth (K-blocks per window). +#ifndef REUSE_MAX +#define REUSE_MAX 15 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per dequant window (cache depth) +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 #define CACHEOP_MAX 0 #define REP_RATE 0 -#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) -#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) +#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) -// L2 SCP layout per minion (double-buffered dequant panel + sync counters). -// panel = BLOCK_K k-lines x TILE_M m (FP32) = 32 * 64 = 2048 bytes, in TenB +// Single dequant panel: BLOCK_K k-lines x TILE_M m (FP32) = 32*64 = 2048 bytes, // [k][m] order: panel[k*TILE_M + m]. -#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t) sizeof(float)) // 2048 -#define SCP_READY_OFF (2 * SCP_PANEL_SIZE) // 4096 -#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 4160 -#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 4224 +#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t)sizeof(float)) // 2048 + +// L2 SCP layout per minion. The REUSE path needs the larger footprint, so the +// per-minion stride uses it for both paths (mutually exclusive at runtime). +// [0 .. RU_BUF_BYTES) cache buffer 0 (KWIN panels) +// [RU_BUF_BYTES .. 2*..) cache buffer 1 (KWIN panels) +// [RU_CACHE_BYTES .. +R*1024) REUSE_MAX C-scratch tiles (16 rows*64B each) +// ready_ctr, consumed_ctr sync counters +// The ORIGINAL path reuses [0,2048) and [2048,4096) as its two panels and the +// same ready/consumed counters (which sit above the cache region). +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // Signal a counter value to the other hart via L2 SCP. -static inline void __attribute__((always_inline)) scp_signal(volatile uint32_t * flag, uint32_t value) { +static inline void __attribute__((always_inline)) +scp_signal(volatile uint32_t *flag, uint32_t value) { *flag = value; FENCE; - evict_to_l2((const void *) flag, 1, 64); + evict_to_l2((const void *)flag, 1, 64); WAIT_CACHEOPS; } // Wait for a counter in L2 SCP to reach the expected value. -static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * flag, uint32_t expected) { +static inline void __attribute__((always_inline)) +scp_wait(volatile uint32_t *flag, uint32_t expected) { while (1) { - evict_to_l2((const void *) flag, 1, 64); + evict_to_l2((const void *)flag, 1, 64); WAIT_CACHEOPS; - if (*flag >= expected) { - return; - } + if (*flag >= expected) return; } } @@ -62,97 +93,129 @@ static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * f // the low/high nibbles to FP32 (nibble-8), scale by the block's fp16 d, and // fscw.ps-scatter the 8 values down 8 panel lines (stride 64B) at column m. // 4 groups of 8 cover the 32 k-values (low 0..15, high 16..31). -static inline void __attribute__((always_inline)) dequant_q4_0_panel(float * panel, - const char * src0_batch, - int64_t mb, - int64_t kb_block, - int64_t nb1_0) { +static inline void __attribute__((always_inline)) +dequant_q4_0_panel(float *panel, const char *src0_batch, + int64_t mb, int64_t kb_block, int64_t nb1_0) { static const int32_t __attribute__((aligned(32))) scatter_idx[8] = { - 0, 64, 128, 192, 256, 320, 384, 448 // byte offsets: 8 lines apart + 0, 64, 128, 192, 256, 320, 384, 448 // byte offsets: 8 lines apart }; static const int32_t __attribute__((aligned(32))) gather_idx[8] = { - 0, 1, 2, 3, 4, 5, 6, 7 // 8 consecutive bytes + 0, 1, 2, 3, 4, 5, 6, 7 // 8 consecutive bytes }; unsigned long old_mask; __asm__ volatile( "mova.x.m %[ms] \n\t" - "mov.m.x m0, x0, 0xFF \n\t" // all 8 lanes active - "flw.ps f1, (%[sidx]) \n\t" // f1 = scatter offsets - "flw.ps f2, (%[gidx]) \n\t" // f2 = gather offsets + "mov.m.x m0, x0, 0xFF \n\t" // all 8 lanes active + "flw.ps f1, (%[sidx]) \n\t" // f1 = scatter offsets + "flw.ps f2, (%[gidx]) \n\t" // f2 = gather offsets : [ms] "=&r"(old_mask) : [sidx] "r"(scatter_idx), [gidx] "r"(gather_idx) - : "f1", "f2"); + : "f1", "f2" + ); - char * pbase = (char *) panel; + char *pbase = (char *) panel; for (int j = 0; j < TILE_M; ++j) { - const block_q4_0 * blk = (const block_q4_0 *) (src0_batch + (mb + j) * nb1_0) + kb_block; - uint32_t scale_raw = (uint32_t) blk->d; - const uint8_t * qs = blk->qs; - char * col = pbase + j * 4; // column m=j of the panel + const block_q4_0 *blk = + (const block_q4_0 *)(src0_batch + (mb + j) * nb1_0) + kb_block; + uint32_t scale_raw = (uint32_t) blk->d; + const uint8_t *qs = blk->qs; + char *col = pbase + j * 4; // column m=j of the panel __asm__ volatile( - "fbcx.ps f3, %[sb] \n\t" // broadcast fp16 scale bits - "fcvt.ps.f16 f3, f3 \n\t" // -> d in all 8 lanes (fp32) + "fbcx.ps f3, %[sb] \n\t" // broadcast fp16 scale bits + "fcvt.ps.f16 f3, f3 \n\t" // -> d in all 8 lanes (fp32) - "fgb.ps f4, f2(%[qs0]) \n\t" // gather qs[0..7] - "fandi.pi f5, f4, 15 \n\t" // low nibble + "fgb.ps f4, f2(%[qs0]) \n\t" // gather qs[0..7] + "fandi.pi f5, f4, 15 \n\t" // low nibble "faddi.pi f5, f5, -8 \n\t" "fcvt.ps.pw f5, f5, rne \n\t" "fmul.ps f5, f5, f3 \n\t" - "fscw.ps f5, f1(%[c0]) \n\t" // k=0..7 -> lines 0..7 - "fsrli.pi f6, f4, 4 \n\t" // high nibble + "fscw.ps f5, f1(%[c0]) \n\t" // k=0..7 -> lines 0..7 + "fsrli.pi f6, f4, 4 \n\t" // high nibble "fandi.pi f6, f6, 15 \n\t" "faddi.pi f6, f6, -8 \n\t" "fcvt.ps.pw f6, f6, rne \n\t" "fmul.ps f6, f6, f3 \n\t" - "fscw.ps f6, f1(%[c16]) \n\t" // k=16..23 -> lines 16..23 + "fscw.ps f6, f1(%[c16]) \n\t" // k=16..23 -> lines 16..23 - "fgb.ps f4, f2(%[qs8]) \n\t" // gather qs[8..15] + "fgb.ps f4, f2(%[qs8]) \n\t" // gather qs[8..15] "fandi.pi f5, f4, 15 \n\t" "faddi.pi f5, f5, -8 \n\t" "fcvt.ps.pw f5, f5, rne \n\t" "fmul.ps f5, f5, f3 \n\t" - "fscw.ps f5, f1(%[c8]) \n\t" // k=8..15 -> lines 8..15 + "fscw.ps f5, f1(%[c8]) \n\t" // k=8..15 -> lines 8..15 "fsrli.pi f6, f4, 4 \n\t" "fandi.pi f6, f6, 15 \n\t" "faddi.pi f6, f6, -8 \n\t" "fcvt.ps.pw f6, f6, rne \n\t" "fmul.ps f6, f6, f3 \n\t" - "fscw.ps f6, f1(%[c24]) \n\t" // k=24..31 -> lines 24..31 + "fscw.ps f6, f1(%[c24]) \n\t" // k=24..31 -> lines 24..31 : - : [sb] "r"(scale_raw), [qs0] "r"(qs), [qs8] "r"(qs + 8), [c0] "r"(col), [c8] "r"(col + 8 * 64), + : [sb] "r"(scale_raw), + [qs0] "r"(qs), [qs8] "r"(qs + 8), + [c0] "r"(col), [c8] "r"(col + 8 * 64), [c16] "r"(col + 16 * 64), [c24] "r"(col + 24 * 64) - : "f3", "f4", "f5", "f6", "memory"); + : "f3", "f4", "f5", "f6", "memory" + ); } - __asm__ volatile("mova.m.x %0" ::"r"(old_mask)); + __asm__ volatile("mova.m.x %0" :: "r"(old_mask)); +} + +// Spill / seed the FP32 C accumulator (16x16 tile in the vector register file, +// row n -> f2n[cols 0..7], f2n+1[cols 8..15]) to/from a 1 KB L2-SCP scratch. +// scratch layout: row n at byte offset n*64. Always moves all 16 rows; rows +// beyond a partial n_cur carry harmless garbage (never stored / recomputed). +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); } -int entry_point(struct ggml_et_binary_params * params, void * env) { +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} + +int entry_point(struct ggml_et_binary_params *params, void *env) { (void) env; uint64_t hart_id = get_hart_id(); uint64_t shire_id = get_shire_id(); - if (shire_id >= NUM_COMPUTE_SHIRES) { - return 0; - } + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; - const int is_hart1 = hart_id & 1; - uint64_t local_minion = (hart_id >> 1) & 0x1F; + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; // Dimensions (both harts need these for tile assignment) const int64_t K = params->src0.ne[0]; const int64_t M = params->src0.ne[1]; const int64_t N = params->src1.ne[1]; - if ((M % TILE_M) != 0) { - return 0; - } - if ((K % BLOCK_K) != 0) { - return 0; - } + if ((M % TILE_M) != 0) return 0; + if ((K % BLOCK_K) != 0) return 0; const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; @@ -166,73 +229,241 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const int64_t nb1_d = params->dst.nb[1]; const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; - const char * src0_base = (const char *) params->src0.data; - const char * src1_base = (const char *) params->src1.data; - char * dst_base = (char *) params->dst.data; + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; - const int64_t m_tiles = M / TILE_M; - const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; const int64_t batch_count = ne2_1 * ne3_1; - const int64_t base_tiles = m_tiles * n_tiles * batch_count; const int64_t r2 = ne2_1 / ne2_0; const int64_t r3 = ne3_1 / ne3_0; - const int64_t k_steps = K / BLOCK_K; // number of Q4_0 blocks + const int64_t k_steps = K / BLOCK_K; // number of Q4_0 blocks - // Force a single K-split. - const int64_t k_splits = 1; - - const int64_t tiles_per_shire = MINIONS_PER_SHIRE / k_splits; - const int64_t k_split = local_minion % k_splits; - const int64_t local_tile_idx = local_minion / k_splits; + const int64_t tiles_per_shire = MINIONS_PER_SHIRE; + const int64_t local_tile_idx = local_minion; const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + // L2 SCP pointers for this minion. + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // Calculate ru_n to perfectly minimize hardware waves while avoiding Consumer bottleneck. + // The pipeline is perfectly balanced at r=8. Score = waves * max(8, r). + // We find the r that minimizes Score. + int64_t best_r = 1; + int64_t min_score = INT64_MAX; + int64_t max_search_r = REUSE_MAX; + if (max_search_r > n_tiles) max_search_r = n_tiles; + + for (int64_t r = 1; r <= max_search_r; r++) { + int64_t n_groups = (n_tiles + r - 1) / r; + int64_t base_units = m_tiles * n_groups * batch_count; + int64_t waves = (base_units + MACHINE_SLOTS - 1) / MACHINE_SLOTS; + + int64_t penalty = (r > 8) ? r : 8; + int64_t score = waves * penalty; + + if (score < min_score) { + min_score = score; + best_r = r; + } + } + int64_t ru_n = best_r; + + // Reuse pays only when it groups >=2 N-tiles; otherwise the windowing / + // C round-trip is pure overhead, so use the one-tile-at-a-time path. + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: dequant each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t n_groups = (n_tiles + ru_n - 1) / ru_n; + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + const int64_t n_windows = (k_steps + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + dequant_q4_0_panel(cf + i * (SCP_PANEL_SIZE / 4), + src0_batch, mb, kb0 + i, nb1_0); + } + FENCE; + flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } - const int64_t k_steps_per_split = k_steps / k_splits; - const int64_t kb_start = k_split * k_steps_per_split; // first block - const int64_t kb_end = kb_start + k_steps_per_split; // one past last + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; // first N-tile + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + const int is_last = (kw == n_windows - 1); + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + if (kw > 0) c_seed(cs); + int first = (kw == 0) ? 1 : 0; + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + for (int64_t i = 0; i < kbn; ++i) { + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = (kb0 + i) * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(cf + i * (SCP_PANEL_SIZE / 4) + half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + } + + if (is_last) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } - // L2 SCP pointers for this minion's double-buffered panels + sync. - uint64_t scp_base = local_minion * SCP_PER_MINION; - float * scp_panel[2] = { + // ===================================================================== + // ORIGINAL path: one output tile at a time (N % TILE_N != 0). No reuse. + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { (float *) et_shire_l2scp_local(scp_base), (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), }; - volatile uint32_t * ready_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); - volatile uint32_t * consumed_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); - // ================================================================ - // Hart 1: Q4_0 weight dequant producer - // ================================================================ if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); - uint32_t chunk_id = 0; - for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; - tile += tiles_stride) { + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { const int64_t tiles_per_batch = m_tiles * n_tiles; const int64_t batch_idx = tile / tiles_per_batch; const int64_t tile_in_batch = tile % tiles_per_batch; - - const int64_t mb_idx = tile_in_batch % m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; const int64_t i3 = batch_idx / ne2_1; const int64_t i2 = batch_idx % ne2_1; const int64_t i2_0 = i2 / r2; const int64_t i3_0 = i3 / r3; - const char * src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; - const int64_t mb = mb_idx * TILE_M; + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; - for (int64_t kb = kb_start; kb < kb_end; ++kb) { + for (int64_t kb = 0; kb < k_steps; ++kb) { int buf = chunk_id & 1; - - // Back-pressure: wait for hart 0 to finish with this buffer. - if (chunk_id >= 2) { - scp_wait(consumed_ctr, chunk_id - 1); - } + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); dequant_q4_0_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); @@ -244,123 +475,79 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { scp_signal(ready_ctr, chunk_id); } } - FENCE; return 0; } - // ================================================================ - // Hart 0: tensor engine compute - // ================================================================ - uint64_t my_minion_id = get_minion_id(); - const uint64_t group_base_global = my_minion_id - k_split; - setup_cache_scp(); #if CACHEOP_MAX > 0 || REP_RATE > 0 ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; - - evict_to_l2((const void *) ready_ctr, 1, 64); - WAIT_CACHEOPS; - evict_to_l2((const void *) consumed_ctr, 1, 64); - WAIT_CACHEOPS; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; uint32_t chunk_id = 0; - - for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; - tile += tiles_stride) { + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { const int64_t tiles_per_batch = m_tiles * n_tiles; const int64_t batch_idx = tile / tiles_per_batch; const int64_t tile_in_batch = tile % tiles_per_batch; - - const int64_t nb_idx = tile_in_batch / m_tiles; - const int64_t mb_idx = tile_in_batch % m_tiles; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; const int64_t i3 = batch_idx / ne2_1; const int64_t i2 = batch_idx % ne2_1; - const char * src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; - char * dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; - const int64_t mb = mb_idx * TILE_M; - const int64_t nb = nb_idx * TILE_N; + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); - - // Partial-N tiles run TensorFMA32 with a_num_rows = n_cur-1. - // Errata Type D workaround for n_cur == 4 (AROWS==3): pad A to AROWS==4. const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); if (n_cur == 4) { - // Zero the padded 5th A row (line A_L1_START+4) once; the per-pass A - // load only writes lines A_L1_START..+3, so this persists. - static const float __attribute__((aligned(64))) zero_line[16] = { 0 }; - tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, (uint64_t) zero_line, 0, - 0, // 1 line - 64, 0); + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); tensor_wait(TENSOR_LOAD_WAIT_0); } - int first = 1; // first_pass=1 only for the very first FMA of the tile - - for (int64_t kb = kb_start; kb < kb_end; ++kb) { + int first = 1; + for (int64_t kb = 0; kb < k_steps; ++kb) { int buf = chunk_id & 1; - - // Wait for hart 1 to finish dequantizing this block. chunk_id++; scp_wait(ready_ctr, chunk_id); - // Two FMA passes over the 32-wide block (16 K-cols each). for (int half = 0; half < 2; ++half) { const int64_t k_elem = kb * BLOCK_K + half * FMA_K; - - // Load A (activations) for this 16-K sub-tile, PLAIN. - tensor_load(false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, - (uint64_t) (src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), 0, n_cur - 1, - (uint64_t) nb1_1, 0); - - // Load B (dequantized weights) half from L2 SCP panel, PLAIN. - tensor_load(false, false, B_L1_START, TENSOR_LOAD_PLAIN, 0, - (uint64_t) (scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), 0, FMA_K - 1, 64, 1); - + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); tensor_wait(TENSOR_LOAD_WAIT_0); - tensor_wait(TENSOR_LOAD_WAIT_1); - tensor_fma(false, - 3, // b_num_col: (16/4)-1 - arows_fma, // a_num_rows (n_cur-1, or 4 for the n_cur==4 errata pad) - FMA_K - 1, // a_num_cols - 0, false, false, false, false, B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_load_setup_b( + false, + (uint64_t)(scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); tensor_wait(TENSOR_FMA_WAIT); first = 0; } - // Signal that this buffer is free for hart 1 to reuse. scp_signal(consumed_ctr, chunk_id); } - // K-split ring reduce. - if (k_splits > 1) { - const uint64_t num_regs = (uint64_t) n_cur * 2; - - if (k_split > 0) { - tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, group_base_global + k_split - 1); - tensor_wait(TENSOR_REDUCE_WAIT); - } - - if (k_split < k_splits - 1) { - tensor_reduce_send(0, num_regs, group_base_global + k_split + 1); - tensor_wait(TENSOR_REDUCE_WAIT); - } - } - - // Store FP32 result tile (only the last k-split owns the final sum). - if (k_split == k_splits - 1) { - tensor_store(0, 0, 3, n_cur - 1, (uint64_t) (dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), 0, - (uint64_t) nb1_d); - tensor_wait(TENSOR_STORE_WAIT); - } + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); } FENCE; diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K.c new file mode 100644 index 000000000000..5ef290d0add1 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K.c @@ -0,0 +1,326 @@ +//****************************************************************************** +// MUL_MAT Kernel (Q4_K weights x F32 activations -> F32) +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +// +// Structure mirrors mul_mat_Q4_0.c. The key difference is the block size: +// Q4_K packs 256 elements per super-block (8 groups of 32) with per-group +// scales/mins, versus 32 elements per block for Q4_0. The actual dequant + +// dot work is delegated to compute_row_dot_q4_K() in block_ops.h. The K-tiling +// and K-split thresholds below are expressed in super-blocks but chosen so the +// element-level behaviour matches the Q4_0 kernel (one Q4_K super-block == 8 +// Q4_0 blocks, so the block thresholds are divided by 8). +//****************************************************************************** + +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "math_fp.h" +#include "quants.h" +#include "block_ops.h" + +#define STRIDE_M 2048 /* 32 shires x 32 minions x 2 harts */ +#define STRIDE_M_KSPLIT 1024 /* 32 shires x 32 minions (both harts share rows) */ +#define KSPLIT_MIN_K_BLOCKS 32 /* K >= 8192 elements (32 super-blocks) */ +#define KSPLIT_SMALL_ROWS_K_BLOCKS 8 /* K >= 2048 elements for very small M */ +#define KSPLIT_MAX_ROWS 8 /* max rows per minion for K-split */ +#define TILE_KB 32 /* K-tile size in Q4_K super-blocks (8192 elems, 32KB B data) */ +#define KSPLIT_GROUP_ROWS 4 + +int entry_point(struct ggml_et_binary_params* params, void* env) { + uint64_t hart_id = get_hart_id(); + + // Matrix dimensions + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + const int64_t ne02 = params->src0.ne[2]; + const int64_t ne03 = params->src0.ne[3]; + const int64_t ne12 = params->src1.ne[2]; + const int64_t ne13 = params->src1.ne[3]; + + // Strides (in bytes) + const size_t nb01 = params->src0.nb[1]; + const size_t nb02 = params->src0.nb[2]; + const size_t nb03 = params->src0.nb[3]; + + const size_t nb11 = params->src1.nb[1]; + const size_t nb12 = params->src1.nb[2]; + const size_t nb13 = params->src1.nb[3]; + + const size_t nbd1 = params->dst.nb[1]; + const size_t nbd2 = params->dst.nb[2]; + const size_t nbd3 = params->dst.nb[3]; + + // Q4_K super-block holds 256 elements + const int64_t K_blocks = K / QK_K; + + // Broadcasting ratios + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // K-split decision + const int64_t minion_id = hart_id >> 1; /* 0..1023 global */ + const int64_t local_minion = (hart_id >> 1) & 0x1F; /* 0..31 within shire */ + const int is_hart1 = hart_id & 1; + const int64_t rows_per_minion = (M + STRIDE_M_KSPLIT - 1) / STRIDE_M_KSPLIT; + const int64_t k_half = K_blocks / 2; + const int use_ksplit_small_rows = (rows_per_minion <= 2) + && (K_blocks >= KSPLIT_SMALL_ROWS_K_BLOCKS); + /* + * K-split when K is large enough to benefit, and either: + * - few rows (≤4): always safe, proven working + * - more rows (5-8): only if each hart's half fits in one tile, + * otherwise L1 thrashing from 2 harts × 8 rows kills performance + * + * Also allow K-split earlier for the low-M regime (≤2 rows/minion). In + * that case the simple row-striped path leaves half the machine idle, so + * using both harts on each row pays off even for moderate K. + */ + const int use_ksplit = ((K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion <= KSPLIT_MAX_ROWS) + && (rows_per_minion <= 4 || k_half <= TILE_KB)) + || use_ksplit_small_rows; + const int use_ksplit_group = !use_ksplit + && (K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion > 4) + && (rows_per_minion <= KSPLIT_MAX_ROWS); + + if (use_ksplit) { + /* Each hart processes half the K dimension */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + + /* One cache-line-aligned L2SCP slot per minion for exchange */ + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = minion_id; m < M; m += STRIDE_M_KSPLIT) { + const block_q4_K* q_row = (const block_q4_K*)(src0_ptr2 + m * nb01); + + float partial = compute_row_dot_q4_K( + q_row + k_start, b_col_base + k_start * QK_K, k_len); + + if (is_hart1) { + *l2scp_slot = partial; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + float other = *l2scp_slot; + et_sem_post(ET_BARRIER_MINION); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, partial + other); + } + } + } + } + } + } else if (use_ksplit_group) { + /* + * Grouped K-split for the 5-8 rows/minion regime. + * + * Both harts process the same 4-row group, each on half of K, and + * exchange 4 partial sums once per group instead of once per row. + */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m_base = minion_id; m_base < M; + m_base += STRIDE_M_KSPLIT * KSPLIT_GROUP_ROWS) { + const int64_t m0 = m_base; + const int64_t m1 = m0 + STRIDE_M_KSPLIT; + const int64_t m2 = m1 + STRIDE_M_KSPLIT; + const int64_t m3 = m2 + STRIDE_M_KSPLIT; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = k_len - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + if (tile_len <= 0) { + break; + } + const float* b_tile = b_col_base + (k_start + kb) * QK_K; + const int64_t row_kb = k_start + kb; + + if (m0 < M) { + s0 += compute_row_dot_q4_K( + (const block_q4_K*)(src0_ptr2 + m0 * nb01) + row_kb, + b_tile, tile_len); + } + if (m1 < M) { + s1 += compute_row_dot_q4_K( + (const block_q4_K*)(src0_ptr2 + m1 * nb01) + row_kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += compute_row_dot_q4_K( + (const block_q4_K*)(src0_ptr2 + m2 * nb01) + row_kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += compute_row_dot_q4_K( + (const block_q4_K*)(src0_ptr2 + m3 * nb01) + row_kb, + b_tile, tile_len); + } + } + + if (is_hart1) { + l2scp_slot[0] = s0; + l2scp_slot[1] = s1; + l2scp_slot[2] = s2; + l2scp_slot[3] = s3; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + const float p0 = l2scp_slot[0]; + const float p1 = l2scp_slot[1]; + const float p2 = l2scp_slot[2]; + const float p3 = l2scp_slot[3]; + et_sem_post(ET_BARRIER_MINION); + + float* c_base = (float*)(dst_ptr2 + n * nbd1); + if (m0 < M) atomic_store_f32((volatile float*)(c_base + m0), s0 + p0); + if (m1 < M) atomic_store_f32((volatile float*)(c_base + m1), s1 + p1); + if (m2 < M) atomic_store_f32((volatile float*)(c_base + m2), s2 + p2); + if (m3 < M) atomic_store_f32((volatile float*)(c_base + m3), s3 + p3); + } + } + } + } + } + } else if (K_blocks > TILE_KB) { + /* + * Tile-outer with scalar row groups: process up to 4 rows per + * hart sharing each B tile before advancing to the next tile. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * 4) { + const int64_t m1 = m0 + STRIDE_M; + const int64_t m2 = m0 + STRIDE_M * 2; + const int64_t m3 = m0 + STRIDE_M * 3; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = K_blocks - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + const float* b_tile = b_col_base + kb * QK_K; + + s0 += compute_row_dot_q4_K( + (const block_q4_K*)(src0_ptr2 + m0 * nb01) + kb, + b_tile, tile_len); + if (m1 < M) { + s1 += compute_row_dot_q4_K( + (const block_q4_K*)(src0_ptr2 + m1 * nb01) + kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += compute_row_dot_q4_K( + (const block_q4_K*)(src0_ptr2 + m2 * nb01) + kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += compute_row_dot_q4_K( + (const block_q4_K*)(src0_ptr2 + m3 * nb01) + kb, + b_tile, tile_len); + } + } + + float* dst_base = (float*)(dst_ptr2 + n * nbd1); + atomic_store_f32((volatile float*)(dst_base + m0), s0); + if (m1 < M) atomic_store_f32((volatile float*)(dst_base + m1), s1); + if (m2 < M) atomic_store_f32((volatile float*)(dst_base + m2), s2); + if (m3 < M) atomic_store_f32((volatile float*)(dst_base + m3), s3); + } + } + } + } + } else { + /* + * Simple path for small K: one row per hart. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q4_K* q_row = (const block_q4_K*)(src0_ptr2 + m * nb01); + + float sum = compute_row_dot_q4_K(q_row, b_col_base, K_blocks); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, sum); + } + } + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c new file mode 100644 index 000000000000..f3abdeba54c8 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c @@ -0,0 +1,595 @@ +//****************************************************************************** +// MUL_MAT Kernel +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +//****************************************************************************** + +#include +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" +#include "quants.h" +#include "math_fp.h" + +// Q4_K x F32 -> F32 MUL_MAT on the tensor (matrix) engine, TensorFMA32. +// identical to mul_mat_Q4_0_matrix_engine.c, identical producer/consumer, tiling and +// tensor-engine loop; only the weight dequant differs (Q4_K affine per-group). +// Hart 1: dequantize Q4_K weights to FP32 into double-buffered L2 SCP. +// Hart 0: tensor engine compute (FMA, reduce, store). +// +// Two execution paths (selected at runtime by N % TILE_N): +// * REUSE path (N % TILE_N == 0): dequantize each weight K-window ONCE and +// reuse it across ru_n consecutive N-tiles, so the (producer-bound) +// dequant work is cut by ~ru_n. Partial C is round-tripped through an +// L2-SCP scratch between K-windows (the FMA C accumulator is a single fixed +// register-file tile, so multiple output tiles cannot be resident at once). +// * ORIGINAL path (N % TILE_N != 0): one output tile at a time, no reuse. + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +#define TILE_M 16 +#define TILE_N 16 +#define BLOCK_K 32 // one Q4_K group (32 elements) per panel +#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) + +// --- Reuse knobs ---------------------------------------------------------- +// REUSE_MAX caps the L2-SCP C-scratch footprint; the actual reuse factor is +// chosen at runtime as the largest value that still keeps the whole +// machine busy. KWIN is the dequant-cache depth (K-blocks per window). +#ifndef REUSE_MAX +#define REUSE_MAX 15 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per dequant window (cache depth) +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 + +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) + +// Single dequant panel: BLOCK_K k-lines x TILE_M m (FP32) = 32*64 = 2048 bytes, +// [k][m] order: panel[k*TILE_M + m]. +#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t)sizeof(float)) // 2048 + +// L2 SCP layout per minion. The REUSE path needs the larger footprint, so the +// per-minion stride uses it for both paths (mutually exclusive at runtime). +// [0 .. RU_BUF_BYTES) cache buffer 0 (KWIN panels) +// [RU_BUF_BYTES .. 2*..) cache buffer 1 (KWIN panels) +// [RU_CACHE_BYTES .. +R*1024) REUSE_MAX C-scratch tiles (16 rows*64B each) +// ready_ctr, consumed_ctr sync counters +// The ORIGINAL path reuses [0,2048) and [2048,4096) as its two panels and the +// same ready/consumed counters (which sit above the cache region). +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) + +// Signal a counter value to the other hart via L2 SCP. +static inline void __attribute__((always_inline)) +scp_signal(volatile uint32_t *flag, uint32_t value) { + *flag = value; + FENCE; + evict_to_l2((const void *)flag, 1, 64); + WAIT_CACHEOPS; +} + +// Wait for a counter in L2 SCP to reach the expected value. +static inline void __attribute__((always_inline)) +scp_wait(volatile uint32_t *flag, uint32_t expected) { + while (1) { + evict_to_l2((const void *)flag, 1, 64); + WAIT_CACHEOPS; + if (*flag >= expected) return; + } +} + +// Software fp16->fp32 (pure integer). The hardware fcvt.ps.f16 returns wrong +// values after the attention block (shared conversion-unit state), which would +// corrupt the Q4_K weight scales here; software conversion avoids that +// instruction entirely. Only 2 conversions per super-block, so cost is negligible. +// Since this is only observed for Q4_K and only after the attention block, we need +// to investigate it further. +static inline float __attribute__((always_inline)) me_sw_fp16(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000) << 16; + uint32_t exp = (h >> 10) & 0x1F; + uint32_t mant = h & 0x3FF; + uint32_t f; + if (exp == 0) { + if (mant == 0) { f = sign; } + else { + exp = 127 - 15 + 1; + while ((mant & 0x400) == 0) { mant <<= 1; exp--; } + mant &= 0x3FF; + f = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1F) { + f = sign | 0x7F800000u | (mant << 13); + } else { + f = sign | ((exp + (127 - 15)) << 23) | (mant << 13); + } + float out; __builtin_memcpy(&out, &f, 4); return out; +} + +// Dequantize one 32-element Q4_K GROUP of TILE_M weight rows into the FP32 panel, +// written in TenB [k][m] order: panel[k*TILE_M + m]. +// +// A Q4_K super-block packs 256 elements as 8 groups of 32, each with a 6-bit +// scale (sc) and 6-bit min (m), plus a super-block fp16 d and dmin: +// w = (d*sc) * nibble - (dmin*m) (nibble in 0..15, no zero point) +// kb_group is the global group index: super-block = kb_group/8, group = kb_group%8. +// Group g draws its 32 nibbles from qs[(g/2)*32 .. +31], low nibble if g even and +// high nibble if g odd (matching dequantize_q4_K_block / compute_row_dot_q4_K). +static inline void __attribute__((always_inline)) +dequant_q4_K_panel(float *panel, const char *src0_batch, + int64_t mb, int64_t kb_group, int64_t nb1_0) { + static const int32_t __attribute__((aligned(32))) scatter_idx[8] = { + 0, 64, 128, 192, 256, 320, 384, 448 // byte offsets: 8 lines apart + }; + static const int32_t __attribute__((aligned(32))) gather_idx[8] = { + 0, 1, 2, 3, 4, 5, 6, 7 // 8 consecutive bytes + }; + + const int64_t sb = kb_group >> 3; // super-block index + const int g = (int) (kb_group & 7); // group within super-block + const int64_t qoff = (int64_t)(g >> 1) * 32; + const int hi = g & 1; // high-nibble group? + + unsigned long old_mask; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" // all 8 lanes active + "flw.ps f1, (%[sidx]) \n\t" // f1 = scatter offsets + "flw.ps f2, (%[gidx]) \n\t" // f2 = gather offsets + : [ms] "=&r"(old_mask) + : [sidx] "r"(scatter_idx), [gidx] "r"(gather_idx) + : "f1", "f2" + ); + + char *pbase = (char *) panel; + for (int j = 0; j < TILE_M; ++j) { + const block_q4_K *blk = + (const block_q4_K *)(src0_batch + (mb + j) * nb1_0) + sb; + const float d = me_sw_fp16(blk->d); + const float dmin = me_sw_fp16(blk->dmin); + uint8_t sc, mm; + get_scale_min_k4(g, blk->scales, &sc, &mm); + const float dsc = d * (float) sc; + const float negmin = -(dmin * (float) mm); + uint32_t dsc_bits, nmin_bits; + __builtin_memcpy(&dsc_bits, &dsc, 4); + __builtin_memcpy(&nmin_bits, &negmin, 4); + + const uint8_t *qs = blk->qs + qoff; // 32 bytes for this group + char *col = pbase + j * 4; // column m=j of the panel + + if (hi) { + __asm__ volatile( + "fbcx.ps f3, %[dsc] \n\t" // (d*sc) in all 8 lanes + "fbcx.ps f7, %[nmin] \n\t" // -(dmin*m) in all 8 lanes + "fgb.ps f4,f2(%[q0]) \n\t fsrli.pi f5,f4,4\n\t fandi.pi f5,f5,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c0]) \n\t" + "fgb.ps f4,f2(%[q8]) \n\t fsrli.pi f5,f4,4\n\t fandi.pi f5,f5,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c8]) \n\t" + "fgb.ps f4,f2(%[q16])\n\t fsrli.pi f5,f4,4\n\t fandi.pi f5,f5,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c16])\n\t" + "fgb.ps f4,f2(%[q24])\n\t fsrli.pi f5,f4,4\n\t fandi.pi f5,f5,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c24])\n\t" + : + : [dsc] "r"(dsc_bits), [nmin] "r"(nmin_bits), + [q0] "r"(qs), [q8] "r"(qs + 8), [q16] "r"(qs + 16), [q24] "r"(qs + 24), + [c0] "r"(col), [c8] "r"(col + 8 * 64), [c16] "r"(col + 16 * 64), [c24] "r"(col + 24 * 64) + : "f3", "f4", "f5", "f7", "memory" + ); + } else { + __asm__ volatile( + "fbcx.ps f3, %[dsc] \n\t" + "fbcx.ps f7, %[nmin] \n\t" + "fgb.ps f4,f2(%[q0]) \n\t fandi.pi f5,f4,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c0]) \n\t" + "fgb.ps f4,f2(%[q8]) \n\t fandi.pi f5,f4,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c8]) \n\t" + "fgb.ps f4,f2(%[q16])\n\t fandi.pi f5,f4,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c16])\n\t" + "fgb.ps f4,f2(%[q24])\n\t fandi.pi f5,f4,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c24])\n\t" + : + : [dsc] "r"(dsc_bits), [nmin] "r"(nmin_bits), + [q0] "r"(qs), [q8] "r"(qs + 8), [q16] "r"(qs + 16), [q24] "r"(qs + 24), + [c0] "r"(col), [c8] "r"(col + 8 * 64), [c16] "r"(col + 16 * 64), [c24] "r"(col + 24 * 64) + : "f3", "f4", "f5", "f7", "memory" + ); + } + } + + __asm__ volatile("mova.m.x %0" :: "r"(old_mask)); +} + +// Spill / seed the FP32 C accumulator (16x16 tile in the vector register file, +// row n -> f2n[cols 0..7], f2n+1[cols 8..15]) to/from a 1 KB L2-SCP scratch. +// scratch layout: row n at byte offset n*64. Always moves all 16 rows; rows +// beyond a partial n_cur carry harmless garbage (never stored / recomputed). +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); +} + +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} + +int entry_point(struct ggml_et_binary_params *params, void *env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + // Dimensions (both harts need these for tile assignment) + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + if ((M % TILE_M) != 0) return 0; + if ((K % QK_K) != 0) return 0; + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t k_steps = K / BLOCK_K; // number of Q4_0 blocks + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE; + const int64_t local_tile_idx = local_minion; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + // L2 SCP pointers for this minion. + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // Calculate ru_n to perfectly minimize hardware waves while avoiding Consumer bottleneck. + // The pipeline is perfectly balanced at r=8. Score = waves * max(8, r). + // We find the r that minimizes Score. + int64_t best_r = 1; + int64_t min_score = INT64_MAX; + int64_t max_search_r = REUSE_MAX; + if (max_search_r > n_tiles) max_search_r = n_tiles; + + for (int64_t r = 1; r <= max_search_r; r++) { + int64_t n_groups = (n_tiles + r - 1) / r; + int64_t base_units = m_tiles * n_groups * batch_count; + int64_t waves = (base_units + MACHINE_SLOTS - 1) / MACHINE_SLOTS; + + int64_t penalty = (r > 8) ? r : 8; + int64_t score = waves * penalty; + + if (score < min_score) { + min_score = score; + best_r = r; + } + } + int64_t ru_n = best_r; + + // Reuse pays only when it groups >=2 N-tiles; otherwise the windowing / + // C round-trip is pure overhead, so use the one-tile-at-a-time path. + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: dequant each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t n_groups = (n_tiles + ru_n - 1) / ru_n; + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + const int64_t n_windows = (k_steps + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + dequant_q4_K_panel(cf + i * (SCP_PANEL_SIZE / 4), + src0_batch, mb, kb0 + i, nb1_0); + } + FENCE; + flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } + + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; // first N-tile + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + const int is_last = (kw == n_windows - 1); + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + if (kw > 0) c_seed(cs); + int first = (kw == 0) ? 1 : 0; + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + for (int64_t i = 0; i < kbn; ++i) { + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = (kb0 + i) * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(cf + i * (SCP_PANEL_SIZE / 4) + half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + } + + if (is_last) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } + + // ===================================================================== + // ORIGINAL path: one output tile at a time (N % TILE_N != 0). No reuse. + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { + (float *) et_shire_l2scp_local(scp_base), + (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), + }; + + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); + + dequant_q4_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_panel[buf], BLOCK_K, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + FENCE; + return 0; + } + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t chunk_id = 0; + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + int first = 1; + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + chunk_id++; + scp_wait(ready_ctr, chunk_id); + + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = kb * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + + scp_signal(consumed_ctr, chunk_id); + } + + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c index 2aab87ad5e52..923acb7b07e0 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c @@ -1,10 +1,9 @@ +#include +#include #include "ggml_tensor.h" #include "platform.h" #include "tensor.h" -#include -#include - // FP16 x FP16 -> FP32 MUL_MAT with hart 1 B-panel packing // // Hart 0: tensor engine (load A, load B from SCP, FMA, reduce, store) @@ -34,26 +33,26 @@ typedef uint16_t et_fp16_t; // [2048..2111] ready counter (hart1 -> hart0, own cache line) // [2112..2175] consumed counter (hart0 -> hart1, own cache line) #define SCP_BPANEL_SIZE (16 * 32 * sizeof(et_fp16_t)) // 1024 bytes -#define SCP_READY_OFF (2 * SCP_BPANEL_SIZE) // 2048 -#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 2112 -#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 2176 +#define SCP_READY_OFF (2 * SCP_BPANEL_SIZE) // 2048 +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 2112 +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 2176 // Signal a counter value to the other hart via L2 SCP. -static inline void __attribute__((always_inline)) scp_signal(volatile uint32_t * flag, uint32_t value) { +static inline void __attribute__((always_inline)) +scp_signal(volatile uint32_t *flag, uint32_t value) { *flag = value; FENCE; - evict_to_l2((const void *) flag, 1, 64); + evict_to_l2((const void *)flag, 1, 64); WAIT_CACHEOPS; } // Wait for a counter in L2 SCP to reach the expected value. -static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * flag, uint32_t expected) { +static inline void __attribute__((always_inline)) +scp_wait(volatile uint32_t *flag, uint32_t expected) { while (1) { - evict_to_l2((const void *) flag, 1, 64); + evict_to_l2((const void *)flag, 1, 64); WAIT_CACHEOPS; - if (*flag >= expected) { - return; - } + if (*flag >= expected) return; } } @@ -66,12 +65,14 @@ static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * f * * Uses fsch.ps scatter store: load 8 pairs per row, scatter to 8 output lines. */ -static inline void __attribute__((always_inline)) pack_b_interleaved(et_fp16_t * out, - const char * src0_batch, - int64_t mb, - int64_t kb, - int64_t nb1_0) { - static const int32_t __attribute__((aligned(32))) scatter_idx[8] = { 0, 64, 128, 192, 256, 320, 384, 448 }; +static inline void __attribute__((always_inline)) +pack_b_interleaved(et_fp16_t *out, + const char *src0_batch, + int64_t mb, int64_t kb, int64_t nb1_0) +{ + static const int32_t __attribute__((aligned(32))) scatter_idx[8] = { + 0, 64, 128, 192, 256, 320, 384, 448 + }; unsigned long old_mask; __asm__ volatile( @@ -80,11 +81,13 @@ static inline void __attribute__((always_inline)) pack_b_interleaved(et_fp16_t * "flw.ps f1, 0(%[idx]) \n\t" : [ms] "=&r"(old_mask) : [idx] "r"(scatter_idx) - : "f1"); + : "f1" + ); for (int j = 0; j < TILE_M; ++j) { - const et_fp16_t * row = (const et_fp16_t *) (src0_batch + (mb + j) * nb1_0) + kb; - char * dst = (char *) out + j * 4; + const et_fp16_t *row = + (const et_fp16_t *)(src0_batch + (mb + j) * nb1_0) + kb; + char *dst = (char *)out + j * 4; __asm__ volatile( "flw.ps f2, 0(%[src]) \n\t" @@ -92,25 +95,30 @@ static inline void __attribute__((always_inline)) pack_b_interleaved(et_fp16_t * "fscw.ps f2, f1(%[d0]) \n\t" "fscw.ps f3, f1(%[d1]) \n\t" : - : [src] "r"(row), [d0] "r"(dst), [d1] "r"(dst + 512) - : "f2", "f3", "memory"); + : [src] "r"(row), + [d0] "r"(dst), + [d1] "r"(dst + 512) + : "f2", "f3", "memory" + ); } - __asm__ volatile("mova.m.x %[ms] \n\t" : : [ms] "r"(old_mask)); + __asm__ volatile( + "mova.m.x %[ms] \n\t" + : + : [ms] "r"(old_mask) + ); } -int entry_point(struct ggml_et_binary_params * params, void * env) { +int entry_point(struct ggml_et_binary_params *params, void *env) { (void) env; uint64_t hart_id = get_hart_id(); uint64_t shire_id = get_shire_id(); - if (shire_id >= NUM_COMPUTE_SHIRES) { - return 0; - } + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; - const int is_hart1 = hart_id & 1; - uint64_t local_minion = (hart_id >> 1) & 0x1F; + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; // Dimensions (both harts need these for tile assignment) const int64_t K = params->src0.ne[0]; @@ -129,31 +137,27 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const int64_t nb1_d = params->dst.nb[1]; const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; - const char * src0_base = (const char *) params->src0.data; - const char * src1_base = (const char *) params->src1.data; - char * dst_base = (char *) params->dst.data; + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; - if ((M % TILE_M) != 0) { - return 0; - } - if ((K % TILE_K) != 0) { - return 0; - } + if ((M % TILE_M) != 0) return 0; + if ((K % TILE_K) != 0) return 0; - const int64_t m_tiles = M / TILE_M; - const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; const int64_t batch_count = ne2_1 * ne3_1; - const int64_t base_tiles = m_tiles * n_tiles * batch_count; + const int64_t base_tiles = m_tiles * n_tiles * batch_count; const int64_t r2 = ne2_1 / ne2_0; const int64_t r3 = ne3_1 / ne3_0; const int64_t total_harts = NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE; - const int64_t k_steps = K / TILE_K; + const int64_t k_steps = K / TILE_K; int64_t k_splits = 1; if (base_tiles < total_harts) { - k_splits = (total_harts + base_tiles - 1) / base_tiles; + k_splits = (total_harts + base_tiles - 1) / base_tiles; int64_t ks = 1; while (ks * 2 <= k_splits && ks * 2 <= 32 && k_steps % (ks * 2) == 0) { ks *= 2; @@ -162,22 +166,24 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { } const int64_t tiles_per_shire = MINIONS_PER_SHIRE / k_splits; - const int64_t k_split = local_minion % k_splits; - const int64_t local_tile_idx = local_minion / k_splits; - const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t k_split = local_minion % k_splits; + const int64_t local_tile_idx = local_minion / k_splits; + const int64_t tiles_stride = (int64_t)NUM_COMPUTE_SHIRES * tiles_per_shire; const int64_t k_steps_per_split = k_steps / k_splits; - const int64_t k_start = k_split * k_steps_per_split * TILE_K; - const int64_t k_end = k_start + k_steps_per_split * TILE_K; + const int64_t k_start = k_split * k_steps_per_split * TILE_K; + const int64_t k_end = k_start + k_steps_per_split * TILE_K; // L2 SCP pointers for this minion's double-buffered panels + sync - uint64_t scp_base = local_minion * SCP_PER_MINION; - et_fp16_t * scp_bp[2] = { - (et_fp16_t *) et_shire_l2scp_local(scp_base), - (et_fp16_t *) et_shire_l2scp_local(scp_base + SCP_BPANEL_SIZE), + uint64_t scp_base = local_minion * SCP_PER_MINION; + et_fp16_t *scp_bp[2] = { + (et_fp16_t *)et_shire_l2scp_local(scp_base), + (et_fp16_t *)et_shire_l2scp_local(scp_base + SCP_BPANEL_SIZE), }; - volatile uint32_t * ready_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); - volatile uint32_t * consumed_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + volatile uint32_t *ready_ctr = + (volatile uint32_t *)et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *)et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); // ================================================================ // Hart 1: B-panel packer @@ -189,8 +195,10 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { uint32_t chunk_id = 0; - for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; + for (int64_t tile = (int64_t)shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; const int64_t batch_idx = tile / tiles_per_batch; const int64_t tile_in_batch = tile % tiles_per_batch; @@ -202,8 +210,8 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const int64_t i2_0 = i2 / r2; const int64_t i3_0 = i3 / r3; - const char * src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; - const int64_t mb = mb_idx * TILE_M; + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { int buf = chunk_id & 1; @@ -231,7 +239,7 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { // ================================================================ // Hart 0: tensor engine compute // ================================================================ - uint64_t my_minion_id = get_minion_id(); + uint64_t my_minion_id = get_minion_id(); const uint64_t group_base_global = my_minion_id - k_split; setup_cache_scp(); @@ -241,15 +249,17 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { CLEAR_TENSOR_ERROR; // Evict any stale L1D copies of sync counters - evict_to_l2((const void *) ready_ctr, 1, 64); + evict_to_l2((const void *)ready_ctr, 1, 64); WAIT_CACHEOPS; - evict_to_l2((const void *) consumed_ctr, 1, 64); + evict_to_l2((const void *)consumed_ctr, 1, 64); WAIT_CACHEOPS; uint32_t chunk_id = 0; - for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; + for (int64_t tile = (int64_t)shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; const int64_t batch_idx = tile / tiles_per_batch; const int64_t tile_in_batch = tile % tiles_per_batch; @@ -257,14 +267,14 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const int64_t nb_idx = tile_in_batch / m_tiles; const int64_t mb_idx = tile_in_batch % m_tiles; - const int64_t i3 = batch_idx / ne2_1; - const int64_t i2 = batch_idx % ne2_1; + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; - const char * src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; - char * dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; - const int64_t mb = mb_idx * TILE_M; - const int64_t nb = nb_idx * TILE_N; + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); // Set tensor_mask for partial N tiles @@ -277,23 +287,54 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { int buf = chunk_id & 1; // Start loading A from DRAM (overlaps with waiting for hart 1) - tensor_load((n_cur < TILE_N), false, A_L1_START, TENSOR_LOAD_PLAIN, 0, - (uint64_t) (src1_batch + nb * nb1_1 + kb * (int64_t) sizeof(et_fp16_t)), 0, n_cur - 1, - (uint64_t) nb1_1, 0); + tensor_load( + (n_cur < TILE_N), false, + A_L1_START, + TENSOR_LOAD_PLAIN, + 0, + (uint64_t)(src1_batch + nb * nb1_1 + kb * (int64_t)sizeof(et_fp16_t)), + 0, + n_cur - 1, + (uint64_t)nb1_1, + 0 + ); // Wait for hart 1 to finish packing this chunk chunk_id++; scp_wait(ready_ctr, chunk_id); // Load B from L2 SCP (hart 1 already flushed it) - tensor_load(false, false, B_L1_START, TENSOR_LOAD_PLAIN, 0, (uint64_t) scp_bp[buf], 0, 15, 64, 1); + tensor_load( + false, false, + B_L1_START, + TENSOR_LOAD_PLAIN, + 0, + (uint64_t)scp_bp[buf], + 0, + 15, + 64, + 1 + ); tensor_wait(TENSOR_LOAD_WAIT_0); tensor_wait(TENSOR_LOAD_WAIT_1); // TensorFMA16A32 - tensor_fma((n_cur < TILE_N), 3, n_cur - 1, 15, 0, false, false, false, false, B_L1_START, A_L1_START, - TENSOR_FMA_OP_FP16, (kb == k_start)); + tensor_fma( + (n_cur < TILE_N), + 3, + n_cur - 1, + 15, + 0, + false, + false, + false, + false, + B_L1_START, + A_L1_START, + TENSOR_FMA_OP_FP16, + (kb == k_start) + ); tensor_wait(TENSOR_FMA_WAIT); @@ -303,23 +344,33 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { // K-split ring reduce if (k_splits > 1) { - const uint64_t num_regs = (uint64_t) n_cur * 2; + const uint64_t num_regs = (uint64_t)n_cur * 2; if (k_split > 0) { - tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, group_base_global + k_split - 1); + tensor_reduce_recv( + 0, TENSOR_REDUCE_OP_FADD, + num_regs, + group_base_global + k_split - 1 + ); tensor_wait(TENSOR_REDUCE_WAIT); } if (k_split < k_splits - 1) { - tensor_reduce_send(0, num_regs, group_base_global + k_split + 1); + tensor_reduce_send( + 0, num_regs, + group_base_global + k_split + 1 + ); tensor_wait(TENSOR_REDUCE_WAIT); } } // Store FP32 result tile if (k_split == k_splits - 1) { - tensor_store(0, 0, 3, n_cur - 1, (uint64_t) (dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), 0, - (uint64_t) nb1_d); + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t)sizeof(float)), + 0, (uint64_t)nb1_d + ); tensor_wait(TENSOR_STORE_WAIT); } } diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c index b2b61d519672..4d14769f5a66 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c @@ -1,10 +1,9 @@ +#include +#include #include "ggml_tensor.h" #include "platform.h" #include "tensor.h" -#include -#include - /* * F32 Matrix Multiply for ET-SoC-1 — TensorFMA32. * @@ -19,22 +18,17 @@ #define TILE_M 16 /* ── Tuning knobs ───────────────────────────────────────────────────── */ -#define TILE_N 16 -#define CACHEOP_MAX 0 -#define REP_RATE 0 - +#define TILE_N 16 +#define CACHEOP_MAX 0 +#define REP_RATE 0 /* ─────────────────────────────────────────────────────────────────── */ -int entry_point(struct ggml_et_binary_params * params, void * env) { - uint64_t hart_id = get_hart_id(); +int entry_point(struct ggml_et_binary_params* params, void* env) { + uint64_t hart_id = get_hart_id(); uint64_t shire_id = get_shire_id(); - if (shire_id >= NUM_COMPUTE_SHIRES) { - return 0; - } - if (hart_id & 1) { - return 0; - } + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + if (hart_id & 1) return 0; uint64_t local_minion = (hart_id >> 1) & 0x1F; uint64_t my_minion_id = get_minion_id(); @@ -51,11 +45,11 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const int64_t nb1_1 = params->src1.nb[1]; const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; const int64_t nb1_d = params->dst.nb[1]; - const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; - const char * src0_base = (const char *) params->src0.data; - const char * src1_base = (const char *) params->src1.data; - char * dst_base = (char *) params->dst.data; + const char* src0_base = (const char*)params->src0.data; + const char* src1_base = (const char*)params->src1.data; + char* dst_base = (char*)params->dst.data; setup_cache_scp(); #if CACHEOP_MAX > 0 || REP_RATE > 0 @@ -63,19 +57,19 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { #endif CLEAR_TENSOR_ERROR; - const int64_t m_tiles = M / TILE_M; - const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; const int64_t batch_count = ne2_1 * ne3_1; - const int64_t base_tiles = m_tiles * n_tiles * batch_count; + const int64_t base_tiles = m_tiles * n_tiles * batch_count; const int64_t r2 = ne2_1 / ne2_0; const int64_t r3 = ne3_1 / ne3_0; const int64_t total_harts = NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE; - const int64_t k_steps = K / TILE_K; - int64_t k_splits = 1; + const int64_t k_steps = K / TILE_K; + int64_t k_splits = 1; if (base_tiles < total_harts) { - k_splits = (total_harts + base_tiles - 1) / base_tiles; + k_splits = (total_harts + base_tiles - 1) / base_tiles; int64_t ks = 1; while (ks * 2 <= k_splits && ks * 2 <= 32 && k_steps % (ks * 2) == 0) { ks *= 2; @@ -84,68 +78,88 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { } const int64_t tiles_per_shire = MINIONS_PER_SHIRE / k_splits; - const int64_t k_split = local_minion % k_splits; - const int64_t local_tile_idx = local_minion / k_splits; - const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t k_split = local_minion % k_splits; + const int64_t local_tile_idx = local_minion / k_splits; + const int64_t tiles_stride = (int64_t)NUM_COMPUTE_SHIRES * tiles_per_shire; const int64_t k_steps_per_split = k_steps / k_splits; - const int64_t k_start = k_split * k_steps_per_split * TILE_K; - const int64_t k_end = k_start + k_steps_per_split * TILE_K; + const int64_t k_start = k_split * k_steps_per_split * TILE_K; + const int64_t k_end = k_start + k_steps_per_split * TILE_K; const uint64_t group_base_global = my_minion_id - k_split; - for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; + for (int64_t tile = (int64_t)shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; - const int64_t batch_idx = tile / tiles_per_batch; - const int64_t tile_in_batch = tile % tiles_per_batch; - const int64_t nb_idx = tile_in_batch / m_tiles; - const int64_t mb_idx = tile_in_batch % m_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; const int64_t i3 = batch_idx / ne2_1; const int64_t i2 = batch_idx % ne2_1; const int64_t i2_0 = i2 / r2; const int64_t i3_0 = i3 / r3; - const char * src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; - const char * src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; - char * dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + const char* src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const char* src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char* dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; - const int64_t mb = mb_idx * TILE_M; - const int64_t nb = nb_idx * TILE_N; + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { - tensor_load(false, false, 0, 0, 0, (uint64_t) (src1_batch + nb * nb1_1 + kb * sizeof(float)), 0, n_cur - 1, - (uint64_t) nb1_1, 0); - tensor_load(false, false, TILE_K, 7, 0, (uint64_t) (src0_batch + mb * nb1_0 + kb * sizeof(float)), 0, - TILE_K - 1, (uint64_t) nb1_0, 1); + tensor_load( + false, false, 0, 0, 0, + (uint64_t)(src1_batch + nb * nb1_1 + kb * sizeof(float)), + 0, n_cur - 1, (uint64_t)nb1_1, 0 + ); + + tensor_load( + false, false, TILE_K, 7, 0, + (uint64_t)(src0_batch + mb * nb1_0 + kb * sizeof(float)), + 0, TILE_K - 1, (uint64_t)nb1_0, 1 + ); tensor_wait(TENSOR_LOAD_WAIT_0); tensor_wait(TENSOR_LOAD_WAIT_1); - tensor_fma(false, 3, n_cur - 1, TILE_K - 1, 0, false, false, false, false, TILE_K, 0, 0, (kb == k_start)); + tensor_fma( + false, 3, n_cur - 1, TILE_K - 1, 0, + false, false, false, false, + TILE_K, 0, 0, + (kb == k_start) + ); tensor_wait(TENSOR_FMA_WAIT); } if (k_splits > 1) { - const uint64_t num_regs = (uint64_t) n_cur * 2; + const uint64_t num_regs = (uint64_t)n_cur * 2; if (k_split > 0) { - tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, group_base_global + k_split - 1); + tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, + num_regs, + group_base_global + k_split - 1); tensor_wait(TENSOR_REDUCE_WAIT); } if (k_split < k_splits - 1) { - tensor_reduce_send(0, num_regs, group_base_global + k_split + 1); + tensor_reduce_send(0, num_regs, + group_base_global + k_split + 1); tensor_wait(TENSOR_REDUCE_WAIT); } } if (k_split == k_splits - 1) { - tensor_store(0, 0, 3, n_cur - 1, (uint64_t) (dst_batch + nb * nb1_d + mb * sizeof(float)), 0, - (uint64_t) nb1_d); + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * sizeof(float)), + 0, (uint64_t)nb1_d + ); tensor_wait(TENSOR_STORE_WAIT); } } diff --git a/ggml/src/ggml-et/et-kernels/src/platform.h b/ggml/src/ggml-et/et-kernels/src/platform.h index 44854052fd64..a36b99a36083 100644 --- a/ggml/src/ggml-et/et-kernels/src/platform.h +++ b/ggml/src/ggml-et/et-kernels/src/platform.h @@ -34,13 +34,14 @@ typedef struct { // Production implementations (like libgcc's __ctzdi2) use optimized bit manipulation // algorithms with lookup tables and parallel bit operations for O(log n) performance. static inline int manual_ctzll(uint64_t x) { - if (x == 0) return 64; - int count = 0; - while ((x & 1) == 0) { - x >>= 1; - count++; - } - return count; + // if (x == 0) return 64; + // int count = 0; + // while ((x & 1) == 0) { + // x >>= 1; + // count++; + // } + // return count; + return 0; } // Manual implementation of population count for bare metal environment @@ -48,12 +49,13 @@ static inline int manual_ctzll(uint64_t x) { // Production implementations (like libgcc's __popcountdi2) use optimized bit-parallel // algorithms with magic constants and bit manipulation tricks for O(1) performance. static inline int manual_popcountll(uint64_t x) { - int count = 0; - while (x) { - count += x & 1; - x >>= 1; - } - return count; + // int count = 0; + // while (x) { + // count += x & 1; + // x >>= 1; + // } + // return count; + return 32; } // Binary GCD (Stein's algorithm) — avoids expensive 64-bit division/remainder. diff --git a/ggml/src/ggml-et/ggml-et-ops.cpp b/ggml/src/ggml-et/ggml-et-ops.cpp index 6c80fe8acde3..4a15fad2b538 100644 --- a/ggml/src/ggml-et/ggml-et-ops.cpp +++ b/ggml/src/ggml-et/ggml-et-ops.cpp @@ -730,7 +730,19 @@ bool ggml_et_op_mul_mat(ggml_backend_et_device_context * dev_ctx, node->src[1]->type == GGML_TYPE_F32) { kernel_name = "mul_mat_Q4_0"; // N < 53, or M % 16 != 0 or K % 32 != 0 src0_type_name = "Q4_0"; - + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q4_K && + node->src[1]->type == GGML_TYPE_F32 && + node->src[1]->ne[1] >= 53 && // N >= 53 (prefill): use matrix engine + node->src[0]->ne[1] % 16 == 0 && // M % TILE_M + node->src[0]->ne[0] % 256 == 0) { // K % QK_K (Q4_K super-block) + kernel_name = "mul_mat_Q4_K_matrix_engine"; + src0_type_name = "Q4_K"; + } else if (node->type == GGML_TYPE_F32 && + node->src[0]->type == GGML_TYPE_Q4_K && + node->src[1]->type == GGML_TYPE_F32) { + + kernel_name = "mul_mat_Q4_K"; // N < 53, or M % 16 != 0, or K % 256 != 0 + src0_type_name = "Q4_K"; } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q8_0 && node->src[1]->type == GGML_TYPE_F32) { kernel_name = "mul_mat_Q8_0"; diff --git a/ggml/src/ggml-et/ggml-et.cpp b/ggml/src/ggml-et/ggml-et.cpp index eafaf677350b..013d3db5adb8 100644 --- a/ggml/src/ggml-et/ggml-et.cpp +++ b/ggml/src/ggml-et/ggml-et.cpp @@ -1016,6 +1016,27 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm supported = src0_first_dim_contiguous && src1_first_dim_contiguous && dst_first_dim_contiguous && dst_properly_ordered; + } else if (op->type == GGML_TYPE_F32 && + op->src[0] && op->src[0]->type == GGML_TYPE_Q4_K && + op->src[1] && op->src[1]->type == GGML_TYPE_F32) { + + // Keep the existing quantized path constraints separate from the + // relaxed non-quant generic fallback. + bool src0_first_dim_contiguous = (op->src[0]->nb[0] == ggml_type_size(op->src[0]->type)); + bool src1_first_dim_contiguous = (op->src[1]->nb[0] == ggml_type_size(op->src[1]->type)); + bool dst_first_dim_contiguous = (op->nb[0] == sizeof(float)); + + bool dst_properly_ordered = true; + for (int d = 0; d < 3; d++) { + if (op->ne[d] > 1 && op->ne[d+1] > 1 && op->nb[d] > op->nb[d+1]) { + dst_properly_ordered = false; + } + } + + supported = src0_first_dim_contiguous && + src1_first_dim_contiguous && + dst_first_dim_contiguous && + dst_properly_ordered; } else { supported = false; } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 084344fb25d7..8435e65da54c 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8696,6 +8696,15 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 1, 3, 2})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 3, 2, 1})); + // [ET] Q4_K MUL_MAT at real Llama-3.2-1B shapes: large K (->use_ksplit path) and + // prefill batch N>=53. These exercise paths the K=256/N<=16 cases above never hit. + for (int64_t n : {1, 64}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 2048, n, 2048, {1, 1}, {1, 1})); // wq/wo + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 512, n, 2048, {1, 1}, {1, 1})); // wk/wv + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 2048, n, 8192, {1, 1}, {1, 1})); // ffn_down + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 8192, n, 2048, {1, 1}, {1, 1})); // ffn_gate/up (simple path, control) + } + for (ggml_type type_a : other_types) { for (ggml_type type_b : {GGML_TYPE_F32}) { if (ggml_blck_size(type_a) != 256) { From c1188785e603d64074c1efd5c45628af7cc1e60a Mon Sep 17 00:00:00 2001 From: Saqib Akram Date: Thu, 16 Jul 2026 20:47:25 +0500 Subject: [PATCH 11/12] =?UTF-8?q?ET=20backend:=20K-quant=20MUL=5FMAT=20(Q2?= =?UTF-8?q?=5FK=20/=20Q3=5FK=20/=20Q5=5FK=20/=20Q6=5FK)=20=E2=80=94=20scal?= =?UTF-8?q?ar,=20tensor-engine=20&=20vectorized=20dots,=20plus=20uberkerne?= =?UTF-8?q?l=20support=20(#16)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds full ET-backend support for the Q2_K, Q3_K, Q5_K, and Q6_K super-block quantizations (previously only Q4_0/Q8_0/Q4_K existed), and substantially improves generation throughput across all K-quants — including the pre-existing Q4_K — by (a) vectorizing the generation dot product and (b) enabling the K-quant matmuls to run inside the uberkernel. For prefill, every K-quant gets a tensor-unit (matrix-engine) kernel delivering very large token/sec. Everything is validated on test-backend-ops (MUL_MAT + GET_ROWS) and by coherent end-to-end generation on Llama-3.2-1B models. --- ggml/src/ggml-et/CMakeLists.txt | 12 + ggml/src/ggml-et/et-kernels/src/block_ops.h | 457 +++++++++++++++ .../src/ggml-et/et-kernels/src/get_rows_f32.c | 92 ++- .../src/ggml-et/et-kernels/src/mul_mat_Q2_K.c | 345 +++++++++++ .../src/mul_mat_Q2_K_matrix_engine.c | 534 +++++++++++++++++ .../src/ggml-et/et-kernels/src/mul_mat_Q3_K.c | 345 +++++++++++ .../src/mul_mat_Q3_K_matrix_engine.c | 536 +++++++++++++++++ .../src/mul_mat_Q4_0_matrix_engine.c | 19 - .../src/ggml-et/et-kernels/src/mul_mat_Q4_K.c | 44 +- .../src/mul_mat_Q4_K_matrix_engine.c | 43 +- .../src/ggml-et/et-kernels/src/mul_mat_Q5_K.c | 345 +++++++++++ .../src/mul_mat_Q5_K_matrix_engine.c | 535 +++++++++++++++++ .../src/ggml-et/et-kernels/src/mul_mat_Q6_K.c | 347 +++++++++++ .../src/mul_mat_Q6_K_matrix_engine.c | 538 ++++++++++++++++++ .../src/mul_mat_f16_matrix_engine.c | 19 - ggml/src/ggml-et/et-kernels/src/platform.h | 23 + ggml/src/ggml-et/et-kernels/src/quants.h | 113 ++++ ggml/src/ggml-et/et-kernels/src/uberkernel.c | 35 ++ ggml/src/ggml-et/ggml-et-ops.cpp | 44 +- ggml/src/ggml-et/ggml-et.cpp | 26 +- 20 files changed, 4377 insertions(+), 75 deletions(-) create mode 100644 ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K.c create mode 100644 ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c create mode 100644 ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K.c create mode 100644 ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c create mode 100644 ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K.c create mode 100644 ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c create mode 100644 ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K.c create mode 100644 ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c diff --git a/ggml/src/ggml-et/CMakeLists.txt b/ggml/src/ggml-et/CMakeLists.txt index 1c89e536257d..a11d0cd3b5e7 100644 --- a/ggml/src/ggml-et/CMakeLists.txt +++ b/ggml/src/ggml-et/CMakeLists.txt @@ -40,6 +40,14 @@ set(KERNELS mul_mat_Q4_0_matrix_engine mul_mat_Q4_K mul_mat_Q4_K_matrix_engine + mul_mat_Q6_K + mul_mat_Q6_K_matrix_engine + mul_mat_Q2_K + mul_mat_Q2_K_matrix_engine + mul_mat_Q3_K + mul_mat_Q3_K_matrix_engine + mul_mat_Q5_K + mul_mat_Q5_K_matrix_engine mul_mat_f16 mul_mat_f16_matrix_engine rope_f32 @@ -106,6 +114,10 @@ set(UBERKERNEL_SUPPORTED_KERNELS mul_mat_id_f32 mul_mat_Q4_0 mul_mat_Q4_K + mul_mat_Q2_K + mul_mat_Q3_K + mul_mat_Q5_K + mul_mat_Q6_K mul_mat_Q8_0 norm_f32 pad_f32 diff --git a/ggml/src/ggml-et/et-kernels/src/block_ops.h b/ggml/src/ggml-et/et-kernels/src/block_ops.h index 5db35244302a..90023172ec46 100644 --- a/ggml/src/ggml-et/et-kernels/src/block_ops.h +++ b/ggml/src/ggml-et/et-kernels/src/block_ops.h @@ -1067,3 +1067,460 @@ static inline float compute_row_dot_q4_K(const block_q4_K* q_row, } return acc; } + +// Vectorized (8-wide) full-row dot for Q4_K. Affine 8-groups-of-32 layout +// w = d*sc*nibble - dmin*m (no qh bit). Group g -> qs pair g/2, nibble +// low(g even)/high(g odd). f10 scale term, f9 min term; result = f10 - f9. +#define Q4V_LO "fandi.pi f12, f11, 15\n\t" +#define Q4V_HI "fsrli.pi f12, f11, 4\n\t fandi.pi f12, f12, 15\n\t" +#define Q4V_CHUNK(NIB, qlp, bp, dscb, dmb) \ + __asm__ volatile( \ + "fgb.ps f11, f31(%[q])\n\t" \ + NIB \ + "fcvt.ps.pw f12, f12, rne\n\t" \ + "fbcx.ps f16, %[dsc]\n\t" \ + "flw.ps f15, 0(%[b])\n\t" \ + "fmul.ps f12, f12, f16\n\t" \ + "fmadd.ps f10, f12, f15, f10\n\t" \ + "fbcx.ps f17, %[dm]\n\t" \ + "fmadd.ps f9, f15, f17, f9\n\t" \ + :: [q] "r"(qlp), [b] "r"(bp), [dsc] "r"(dscb), [dm] "r"(dmb) \ + : "f11", "f12", "f15", "f16", "f17") +#define Q4V_GROUP(NIB, pp) \ + do { \ + const uint8_t* qlb = block->qs + (pp) * 32; \ + Q4V_CHUNK(NIB, qlb + 0, bg + 0, dscb, dmb); \ + Q4V_CHUNK(NIB, qlb + 8, bg + 8, dscb, dmb); \ + Q4V_CHUNK(NIB, qlb + 16, bg + 16, dscb, dmb); \ + Q4V_CHUNK(NIB, qlb + 24, bg + 24, dscb, dmb); \ + } while (0) + +static inline float compute_row_dot_q4_K_vec(const block_q4_K* q_row, + const float* b_col, + int64_t K_sblocks) { + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + static const int32_t __attribute__((aligned(64))) gp[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + __asm__ volatile("flw.ps f31, %[g]\n\t" + "fbci.ps f10, 0\n\t" + "fbci.ps f9, 0\n\t" + :: [g] "m"(*(const int32_t(*)[8]) gp) + : "f31", "f10", "f9"); + + for (int64_t sb = 0; sb < K_sblocks; sb++) { + const block_q4_K* block = q_row + sb; + const float* b = b_col + sb * QK_K; + const float d = sw_fp16_to_fp32(block->d); + const float min = sw_fp16_to_fp32(block->dmin); + + for (int g = 0; g < 8; ++g) { + uint8_t sc, m; + get_scale_min_k4(g, block->scales, &sc, &m); + const float dscf = d * (float) sc; + const float dmf = min * (float) m; + uint32_t dscb, dmb; + __builtin_memcpy(&dscb, &dscf, 4); + __builtin_memcpy(&dmb, &dmf, 4); + const float* bg = b + g * 32; + const int p = g >> 1; + + if (g & 1) { + Q4V_GROUP(Q4V_HI, p); + } else { + Q4V_GROUP(Q4V_LO, p); + } + } + } + + float final_sum; + __asm__ volatile( + "fswizz.ps f1, f10, 0xB1 \n\t fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps f6, f4, f5, rne \n\t" + "fswizz.ps f1, f9, 0xB1 \n\t fadd.ps f2, f9, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps f7, f4, f5, rne \n\t" + "fsub.ps %[out], f6, f7, rne \n\t" + : [out] "=f"(final_sum) + :: "t0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f9", "f10"); + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return final_sum; +} + +// Vectorized (8-wide) full-row dot for Q6_K, modeled on the q4_0/q8_0 vector +// dots. Each 32-element group of contiguous output elements maps to contiguous +// ql/qh bytes and a fixed nibble/shift, so a group is 4 chunks of 8 lanes. +// The per-group factor d*scale is folded into the weight vector (fmul), so a +// single fp32 accumulator (f10) covers the whole row and is reduced once at the +// end. f30 holds -32.0 (the Q6_K zero point), f31 the byte-gather pattern. +#define Q6V_NIB_LO "fandi.pi f12, f11, 15\n\t" +#define Q6V_NIB_HI "fsrli.pi f12, f11, 4\n\t fandi.pi f12, f12, 15\n\t" +#define Q6V_CHUNK(NIB, SH, qlp, qhp, bp, facbits) \ + __asm__ volatile( \ + "fgb.ps f11, f31(%[q])\n\t" \ + "fgb.ps f13, f31(%[h])\n\t" \ + NIB \ + "fsrli.pi f13, f13, " #SH "\n\t" \ + "fandi.pi f13, f13, 3\n\t" \ + "fslli.pi f13, f13, 4\n\t" \ + "fcvt.ps.pw f12, f12, rne\n\t" \ + "fcvt.ps.pw f13, f13, rne\n\t" \ + "fadd.ps f12, f12, f13, rne\n\t" \ + "fadd.ps f12, f12, f30, rne\n\t" \ + "fbcx.ps f16, %[f]\n\t" \ + "flw.ps f15, 0(%[b])\n\t" \ + "fmul.ps f12, f12, f16\n\t" \ + "fmadd.ps f10, f12, f15, f10\n\t" \ + :: [q] "r"(qlp), [h] "r"(qhp), [b] "r"(bp), [f] "r"(facbits) \ + : "f11", "f12", "f13", "f15", "f16") + +static inline float compute_row_dot_q6_K_vec(const block_q6_K* q_row, + const float* b_col, + int64_t K_sblocks) { + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + static const int32_t __attribute__((aligned(64))) gp[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + const uint32_t neg32 = 0xC2000000u; // -32.0f + __asm__ volatile("flw.ps f31, %[g]\n\t" + "fbcx.ps f30, %[n]\n\t" + "fbci.ps f10, 0\n\t" + :: [g] "m"(*(const int32_t(*)[8]) gp), [n] "r"(neg32) + : "f31", "f30", "f10"); + + for (int64_t sb = 0; sb < K_sblocks; sb++) { + const block_q6_K* block = q_row + sb; + const float* b = b_col + sb * QK_K; + const float d = sw_fp16_to_fp32(block->d); + + for (int g = 0; g < 8; ++g) { + const int c = g >> 2; + const int sub = g & 3; + const uint8_t* qlbase = block->ql + 64 * c + (sub & 1) * 32; + const uint8_t* qhbase = block->qh + 32 * c; + const float* bg = b + g * 32; + const float f0f = d * (float) block->scales[8 * c + 2 * sub + 0]; + const float f1f = d * (float) block->scales[8 * c + 2 * sub + 1]; + uint32_t f0, f1; + __builtin_memcpy(&f0, &f0f, 4); + __builtin_memcpy(&f1, &f1f, 4); + + switch (sub) { + case 0: + Q6V_CHUNK(Q6V_NIB_LO, 0, qlbase + 0, qhbase + 0, bg + 0, f0); + Q6V_CHUNK(Q6V_NIB_LO, 0, qlbase + 8, qhbase + 8, bg + 8, f0); + Q6V_CHUNK(Q6V_NIB_LO, 0, qlbase + 16, qhbase + 16, bg + 16, f1); + Q6V_CHUNK(Q6V_NIB_LO, 0, qlbase + 24, qhbase + 24, bg + 24, f1); + break; + case 1: + Q6V_CHUNK(Q6V_NIB_LO, 2, qlbase + 0, qhbase + 0, bg + 0, f0); + Q6V_CHUNK(Q6V_NIB_LO, 2, qlbase + 8, qhbase + 8, bg + 8, f0); + Q6V_CHUNK(Q6V_NIB_LO, 2, qlbase + 16, qhbase + 16, bg + 16, f1); + Q6V_CHUNK(Q6V_NIB_LO, 2, qlbase + 24, qhbase + 24, bg + 24, f1); + break; + case 2: + Q6V_CHUNK(Q6V_NIB_HI, 4, qlbase + 0, qhbase + 0, bg + 0, f0); + Q6V_CHUNK(Q6V_NIB_HI, 4, qlbase + 8, qhbase + 8, bg + 8, f0); + Q6V_CHUNK(Q6V_NIB_HI, 4, qlbase + 16, qhbase + 16, bg + 16, f1); + Q6V_CHUNK(Q6V_NIB_HI, 4, qlbase + 24, qhbase + 24, bg + 24, f1); + break; + default: + Q6V_CHUNK(Q6V_NIB_HI, 6, qlbase + 0, qhbase + 0, bg + 0, f0); + Q6V_CHUNK(Q6V_NIB_HI, 6, qlbase + 8, qhbase + 8, bg + 8, f0); + Q6V_CHUNK(Q6V_NIB_HI, 6, qlbase + 16, qhbase + 16, bg + 16, f1); + Q6V_CHUNK(Q6V_NIB_HI, 6, qlbase + 24, qhbase + 24, bg + 24, f1); + break; + } + } + } + + float final_sum; + __asm__ volatile( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[out], f4, f5, rne \n\t" + : [out] "=f"(final_sum) + :: "t0", "f1", "f2", "f3", "f4", "f5", "f10"); + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return final_sum; +} + +// NOTE: two further Q6_K dot experiments were tried and REMOVED as neither +// helped - the generation dot is memory/overhead bound, not compute bound: +// * gather-reduced (ql/qh gathered once, reused across sub-groups, 64->24 +// fgb/super-block) - perf-neutral (14.95 vs 14.93 t/s). +// * 4 rotating accumulators to pipeline the fmadd chain - also neutral (15.02). + +// Vectorized (8-wide) full-row dot for Q5_K. Same 8-groups-of-32 affine layout +// as Q4_K plus a 5th bit from qh: w = d*sc*(nibble + qh_bit*16) - dmin*m. +// Group g -> qs pair g/2, nibble low(g even)/high(g odd), qh bit at position g. +// f10 accumulates the scale term, f9 the min term (sum of b*dmin*m per group); +// result = reduce(f10) - reduce(f9). +#define Q5V_LO "fandi.pi f12, f11, 15\n\t" +#define Q5V_HI "fsrli.pi f12, f11, 4\n\t fandi.pi f12, f12, 15\n\t" +#define Q5V_CHUNK(NIB, BP, qlp, qhp, bp, dscb, dmb) \ + __asm__ volatile( \ + "fgb.ps f11, f31(%[q])\n\t" \ + "fgb.ps f13, f31(%[h])\n\t" \ + NIB \ + "fsrli.pi f13, f13, " #BP "\n\t" \ + "fandi.pi f13, f13, 1\n\t" \ + "fslli.pi f13, f13, 4\n\t" \ + "fcvt.ps.pw f12, f12, rne\n\t" \ + "fcvt.ps.pw f13, f13, rne\n\t" \ + "fadd.ps f12, f12, f13, rne\n\t" \ + "fbcx.ps f16, %[dsc]\n\t" \ + "flw.ps f15, 0(%[b])\n\t" \ + "fmul.ps f12, f12, f16\n\t" \ + "fmadd.ps f10, f12, f15, f10\n\t" \ + "fbcx.ps f17, %[dm]\n\t" \ + "fmadd.ps f9, f15, f17, f9\n\t" \ + :: [q] "r"(qlp), [h] "r"(qhp), [b] "r"(bp), [dsc] "r"(dscb), [dm] "r"(dmb) \ + : "f11", "f12", "f13", "f15", "f16", "f17") +#define Q5V_GROUP(NIB, BP, pp) \ + do { \ + const uint8_t* qlb = block->qs + (pp) * 32; \ + const uint8_t* qhb = block->qh; \ + Q5V_CHUNK(NIB, BP, qlb + 0, qhb + 0, bg + 0, dscb, dmb); \ + Q5V_CHUNK(NIB, BP, qlb + 8, qhb + 8, bg + 8, dscb, dmb); \ + Q5V_CHUNK(NIB, BP, qlb + 16, qhb + 16, bg + 16, dscb, dmb); \ + Q5V_CHUNK(NIB, BP, qlb + 24, qhb + 24, bg + 24, dscb, dmb); \ + } while (0) + +static inline float compute_row_dot_q5_K_vec(const block_q5_K* q_row, + const float* b_col, + int64_t K_sblocks) { + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + static const int32_t __attribute__((aligned(64))) gp[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + __asm__ volatile("flw.ps f31, %[g]\n\t" + "fbci.ps f10, 0\n\t" + "fbci.ps f9, 0\n\t" + :: [g] "m"(*(const int32_t(*)[8]) gp) + : "f31", "f10", "f9"); + + for (int64_t sb = 0; sb < K_sblocks; sb++) { + const block_q5_K* block = q_row + sb; + const float* b = b_col + sb * QK_K; + const float d = sw_fp16_to_fp32(block->d); + const float min = sw_fp16_to_fp32(block->dmin); + + for (int g = 0; g < 8; ++g) { + uint8_t sc, m; + get_scale_min_k4(g, block->scales, &sc, &m); + const float dscf = d * (float) sc; + const float dmf = min * (float) m; + uint32_t dscb, dmb; + __builtin_memcpy(&dscb, &dscf, 4); + __builtin_memcpy(&dmb, &dmf, 4); + const float* bg = b + g * 32; + const int p = g >> 1; + + switch (g) { + case 0: Q5V_GROUP(Q5V_LO, 0, p); break; + case 1: Q5V_GROUP(Q5V_HI, 1, p); break; + case 2: Q5V_GROUP(Q5V_LO, 2, p); break; + case 3: Q5V_GROUP(Q5V_HI, 3, p); break; + case 4: Q5V_GROUP(Q5V_LO, 4, p); break; + case 5: Q5V_GROUP(Q5V_HI, 5, p); break; + case 6: Q5V_GROUP(Q5V_LO, 6, p); break; + default: Q5V_GROUP(Q5V_HI, 7, p); break; + } + } + } + + float final_sum; + __asm__ volatile( + "fswizz.ps f1, f10, 0xB1 \n\t fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps f6, f4, f5, rne \n\t" + "fswizz.ps f1, f9, 0xB1 \n\t fadd.ps f2, f9, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps f7, f4, f5, rne \n\t" + "fsub.ps %[out], f6, f7, rne \n\t" + : [out] "=f"(final_sum) + :: "t0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f9", "f10"); + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return final_sum; +} + +// Vectorized (8-wide) full-row dot for Q2_K. Affine 2-bit +// w = d*(sc&0xF)*q - dmin*(sc>>4), 16 groups of 16. Sub-group sg -> chunk +// c=sg/8, shift 2*((sg&7)/2), A/B nibble-set ab=(sg&7)&1 selecting qs/scale. +// f10 scale term, f9 min term; result = reduce(f10) - reduce(f9). +#define Q2V_CHUNK(SH, qp, bp, dlb, mlb) \ + __asm__ volatile( \ + "fgb.ps f11, f31(%[q])\n\t" \ + "fsrli.pi f12, f11, " #SH "\n\t" \ + "fandi.pi f12, f12, 3\n\t" \ + "fcvt.ps.pw f12, f12, rne\n\t" \ + "fbcx.ps f16, %[dl]\n\t" \ + "flw.ps f15, 0(%[b])\n\t" \ + "fmul.ps f12, f12, f16\n\t" \ + "fmadd.ps f10, f12, f15, f10\n\t" \ + "fbcx.ps f17, %[ml]\n\t" \ + "fmadd.ps f9, f15, f17, f9\n\t" \ + :: [q] "r"(qp), [b] "r"(bp), [dl] "r"(dlb), [ml] "r"(mlb) \ + : "f11", "f12", "f15", "f16", "f17") +#define Q2V_GROUP(SH, qb, bg) \ + do { \ + Q2V_CHUNK(SH, (qb) + 0, (bg) + 0, dlb, mlb); \ + Q2V_CHUNK(SH, (qb) + 8, (bg) + 8, dlb, mlb); \ + } while (0) + +static inline float compute_row_dot_q2_K_vec(const block_q2_K* q_row, + const float* b_col, + int64_t K_sblocks) { + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + static const int32_t __attribute__((aligned(64))) gp[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + __asm__ volatile("flw.ps f31, %[g]\n\t" + "fbci.ps f10, 0\n\t" + "fbci.ps f9, 0\n\t" + :: [g] "m"(*(const int32_t(*)[8]) gp) + : "f31", "f10", "f9"); + + for (int64_t sb = 0; sb < K_sblocks; sb++) { + const block_q2_K* block = q_row + sb; + const float* b = b_col + sb * QK_K; + const float d = sw_fp16_to_fp32(block->d); + const float min = sw_fp16_to_fp32(block->dmin); + + for (int sg = 0; sg < 16; ++sg) { + const int c = sg >> 3; + const int sgi = sg & 7; + const int j = sgi >> 1; + const int ab = sgi & 1; + const uint8_t sc = block->scales[c * 8 + 2 * j + ab]; + const float dlf = d * (float) (sc & 0xF); + const float mlf = min * (float) (sc >> 4); + uint32_t dlb, mlb; + __builtin_memcpy(&dlb, &dlf, 4); + __builtin_memcpy(&mlb, &mlf, 4); + const uint8_t* qb = block->qs + c * 32 + ab * 16; + const float* bg = b + sg * 16; + + switch (j) { + case 0: Q2V_GROUP(0, qb, bg); break; + case 1: Q2V_GROUP(2, qb, bg); break; + case 2: Q2V_GROUP(4, qb, bg); break; + default: Q2V_GROUP(6, qb, bg); break; + } + } + } + + float final_sum; + __asm__ volatile( + "fswizz.ps f1, f10, 0xB1 \n\t fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps f6, f4, f5, rne \n\t" + "fswizz.ps f1, f9, 0xB1 \n\t fadd.ps f2, f9, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps f7, f4, f5, rne \n\t" + "fsub.ps %[out], f6, f7, rne \n\t" + : [out] "=f"(final_sum) + :: "t0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f9", "f10"); + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return final_sum; +} + +// Vectorized (8-wide) full-row dot for Q3_K. 3-bit +// w = d*(scale-32)*(q2 + hbit*4 - 4), 16 groups of 16, no min term. Sub-group +// sg -> c=sg/8, j=(sg&7)/2 (qs shift 2*j), ab=(sg&7)&1 (qs/hmask offset), +// hmask bit at position c*4+j. f30 holds -4.0. +#define Q3V_CHUNK(SH, BP, qp, hp, bp, dlb) \ + __asm__ volatile( \ + "fgb.ps f11, f31(%[q])\n\t" \ + "fgb.ps f13, f31(%[h])\n\t" \ + "fsrli.pi f12, f11, " #SH "\n\t fandi.pi f12, f12, 3\n\t" \ + "fsrli.pi f13, f13, " #BP "\n\t fandi.pi f13, f13, 1\n\t fslli.pi f13, f13, 2\n\t" \ + "fcvt.ps.pw f12, f12, rne\n\t" \ + "fcvt.ps.pw f13, f13, rne\n\t" \ + "fadd.ps f12, f12, f13, rne\n\t" \ + "fadd.ps f12, f12, f30, rne\n\t" \ + "fbcx.ps f16, %[dl]\n\t" \ + "flw.ps f15, 0(%[b])\n\t" \ + "fmul.ps f12, f12, f16\n\t" \ + "fmadd.ps f10, f12, f15, f10\n\t" \ + :: [q] "r"(qp), [h] "r"(hp), [b] "r"(bp), [dl] "r"(dlb) \ + : "f11", "f12", "f13", "f15", "f16") +#define Q3V_GROUP(SH, BP, qb, hb, bg) \ + do { \ + Q3V_CHUNK(SH, BP, (qb) + 0, (hb) + 0, (bg) + 0, dlb); \ + Q3V_CHUNK(SH, BP, (qb) + 8, (hb) + 8, (bg) + 8, dlb); \ + } while (0) + +static inline float compute_row_dot_q3_K_vec(const block_q3_K* q_row, + const float* b_col, + int64_t K_sblocks) { + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + static const int32_t __attribute__((aligned(64))) gp[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + const uint32_t neg4 = 0xC0800000u; // -4.0f + __asm__ volatile("flw.ps f31, %[g]\n\t" + "fbcx.ps f30, %[n]\n\t" + "fbci.ps f10, 0\n\t" + :: [g] "m"(*(const int32_t(*)[8]) gp), [n] "r"(neg4) + : "f31", "f30", "f10"); + + for (int64_t sb = 0; sb < K_sblocks; sb++) { + const block_q3_K* block = q_row + sb; + const float* b = b_col + sb * QK_K; + const float d = sw_fp16_to_fp32(block->d); + int8_t scales[16]; + unpack_q3_K_scales(block->scales, scales); + + for (int sg = 0; sg < 16; ++sg) { + const int c = sg >> 3; + const int sgi = sg & 7; + const int j = sgi >> 1; + const int ab = sgi & 1; + const float dlf = d * (float) (scales[c * 8 + 2 * j + ab] - 32); + uint32_t dlb; + __builtin_memcpy(&dlb, &dlf, 4); + const uint8_t* qb = block->qs + c * 32 + ab * 16; + const uint8_t* hb = block->hmask + ab * 16; + const float* bg = b + sg * 16; + const int cj = c * 4 + j; // hmask bit position + + switch (cj) { + case 0: Q3V_GROUP(0, 0, qb, hb, bg); break; + case 1: Q3V_GROUP(2, 1, qb, hb, bg); break; + case 2: Q3V_GROUP(4, 2, qb, hb, bg); break; + case 3: Q3V_GROUP(6, 3, qb, hb, bg); break; + case 4: Q3V_GROUP(0, 4, qb, hb, bg); break; + case 5: Q3V_GROUP(2, 5, qb, hb, bg); break; + case 6: Q3V_GROUP(4, 6, qb, hb, bg); break; + default: Q3V_GROUP(6, 7, qb, hb, bg); break; + } + } + } + + float final_sum; + __asm__ volatile( + "fswizz.ps f1, f10, 0xB1 \n\t fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps %[out], f4, f5, rne \n\t" + : [out] "=f"(final_sum) + :: "t0", "f1", "f2", "f3", "f4", "f5", "f10"); + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return final_sum; +} diff --git a/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c b/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c index 701f1db98e0e..db69ebba2885 100644 --- a/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +++ b/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c @@ -137,6 +137,70 @@ static void copy_q4_K_row(float * dst, const block_q4_K * src_blocks, int64_t nu } } +// Copy a row of Q2_K data to F32 destination (with dequantization) +static void copy_q2_K_row(float * dst, const block_q2_K * src_blocks, int64_t num_elements) { + const int64_t num_blocks = (num_elements + QK_K - 1) / QK_K; + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const int64_t elements_in_block = (block_idx == num_blocks - 1) ? (num_elements - block_idx * QK_K) : QK_K; + + float temp_buffer[QK_K]; + dequantize_q2_K_block(&src_blocks[block_idx], temp_buffer); + + for (int64_t i = 0; i < elements_in_block; i++) { + dst[block_idx * QK_K + i] = temp_buffer[i]; + } + } +} + +// Copy a row of Q3_K data to F32 destination (with dequantization) +static void copy_q3_K_row(float * dst, const block_q3_K * src_blocks, int64_t num_elements) { + const int64_t num_blocks = (num_elements + QK_K - 1) / QK_K; + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const int64_t elements_in_block = (block_idx == num_blocks - 1) ? (num_elements - block_idx * QK_K) : QK_K; + + float temp_buffer[QK_K]; + dequantize_q3_K_block(&src_blocks[block_idx], temp_buffer); + + for (int64_t i = 0; i < elements_in_block; i++) { + dst[block_idx * QK_K + i] = temp_buffer[i]; + } + } +} + +// Copy a row of Q5_K data to F32 destination (with dequantization) +static void copy_q5_K_row(float * dst, const block_q5_K * src_blocks, int64_t num_elements) { + const int64_t num_blocks = (num_elements + QK_K - 1) / QK_K; + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const int64_t elements_in_block = (block_idx == num_blocks - 1) ? (num_elements - block_idx * QK_K) : QK_K; + + float temp_buffer[QK_K]; + dequantize_q5_K_block(&src_blocks[block_idx], temp_buffer); + + for (int64_t i = 0; i < elements_in_block; i++) { + dst[block_idx * QK_K + i] = temp_buffer[i]; + } + } +} + +// Copy a row of Q6_K data to F32 destination (with dequantization) +static void copy_q6_K_row(float * dst, const block_q6_K * src_blocks, int64_t num_elements) { + const int64_t num_blocks = (num_elements + QK_K - 1) / QK_K; + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const int64_t elements_in_block = (block_idx == num_blocks - 1) ? (num_elements - block_idx * QK_K) : QK_K; + + float temp_buffer[QK_K]; + dequantize_q6_K_block(&src_blocks[block_idx], temp_buffer); + + for (int64_t i = 0; i < elements_in_block; i++) { + dst[block_idx * QK_K + i] = temp_buffer[i]; + } + } +} + static void dequantize_q8_0_block_cache_aligned(const block_q8_0 * block, float * dst) { const int8_t * qs_ptr = block->qs; @@ -521,7 +585,8 @@ int entry_point(struct ggml_et_get_rows_params * params, void * env) { } if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_Q8_0 && - src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K) { + src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q6_K && + src0->type != GGML_TYPE_Q2_K && src0->type != GGML_TYPE_Q3_K && src0->type != GGML_TYPE_Q5_K) { return -1; // Unsupported input type } @@ -605,6 +670,31 @@ int entry_point(struct ggml_et_get_rows_params * params, void * env) { const block_q4_K * src_blocks = (const block_q4_K *) src0_data + src_block_offset; float * dst_row = dst_data + dst_offset * ne00; copy_q4_K_row(dst_row, src_blocks, ne00); + } else if (src0->type == GGML_TYPE_Q6_K) { + // Q6_K source: dequantize while copying + const int64_t blocks_per_row = (ne00 + QK_K - 1) / QK_K; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const block_q6_K * src_blocks = (const block_q6_K *) src0_data + src_block_offset; + float * dst_row = dst_data + dst_offset * ne00; + copy_q6_K_row(dst_row, src_blocks, ne00); + } else if (src0->type == GGML_TYPE_Q2_K) { + const int64_t blocks_per_row = (ne00 + QK_K - 1) / QK_K; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const block_q2_K * src_blocks = (const block_q2_K *) src0_data + src_block_offset; + float * dst_row = dst_data + dst_offset * ne00; + copy_q2_K_row(dst_row, src_blocks, ne00); + } else if (src0->type == GGML_TYPE_Q3_K) { + const int64_t blocks_per_row = (ne00 + QK_K - 1) / QK_K; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const block_q3_K * src_blocks = (const block_q3_K *) src0_data + src_block_offset; + float * dst_row = dst_data + dst_offset * ne00; + copy_q3_K_row(dst_row, src_blocks, ne00); + } else if (src0->type == GGML_TYPE_Q5_K) { + const int64_t blocks_per_row = (ne00 + QK_K - 1) / QK_K; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const block_q5_K * src_blocks = (const block_q5_K *) src0_data + src_block_offset; + float * dst_row = dst_data + dst_offset * ne00; + copy_q5_K_row(dst_row, src_blocks, ne00); } } diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K.c new file mode 100644 index 000000000000..9a8323f3f38f --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K.c @@ -0,0 +1,345 @@ +//****************************************************************************** +// MUL_MAT Kernel (Q2_K weights x F32 activations -> F32) +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +// +// Structure mirrors mul_mat_Q4_K.c. Q2_K also packs 256 elements per +// super-block (16 groups of 16) with per-group int8 scales, so the same +// super-block K-tiling applies; only the per-element dequant differs and is +// delegated to Q2K_DOT() in block_ops.h. +//****************************************************************************** + +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "math_fp.h" +#include "quants.h" +#include "block_ops.h" + +#define STRIDE_M 2048 /* 32 shires x 32 minions x 2 harts */ +#define STRIDE_M_KSPLIT 1024 /* 32 shires x 32 minions (both harts share rows) */ +#define KSPLIT_MIN_K_BLOCKS 32 /* K >= 8192 elements (32 super-blocks) */ +#define KSPLIT_SMALL_ROWS_K_BLOCKS 8 /* K >= 2048 elements for very small M */ +#define KSPLIT_MAX_ROWS 8 /* max rows per minion for K-split */ +#define TILE_KB 32 /* K-tile size in Q2_K super-blocks (8192 elems) */ +#define KSPLIT_GROUP_ROWS 4 + +// Vectorized (8-wide) dot. Scalar kept for A/B. +#define Q2K_DOT(a, b, c) compute_row_dot_q2_K_vec(a, b, c) + +#ifdef ET_UBERKERNEL +static inline size_t tensor_bytes(const struct ggml_tensor* t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} +#endif + +int entry_point(struct ggml_et_binary_params* params, void* env) { + uint64_t hart_id = get_hart_id(); + +#ifdef ET_UBERKERNEL + // Uberkernel coherency: src1 (activations) may be stale in L1/L2; force + // re-read from L3/DRAM. src0 (weights) is read-only, never stale. + evict_region_past_l2(params->src1.data, tensor_bytes(¶ms->src1)); +#endif + + // Matrix dimensions + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + const int64_t ne02 = params->src0.ne[2]; + const int64_t ne03 = params->src0.ne[3]; + const int64_t ne12 = params->src1.ne[2]; + const int64_t ne13 = params->src1.ne[3]; + + // Strides (in bytes) + const size_t nb01 = params->src0.nb[1]; + const size_t nb02 = params->src0.nb[2]; + const size_t nb03 = params->src0.nb[3]; + + const size_t nb11 = params->src1.nb[1]; + const size_t nb12 = params->src1.nb[2]; + const size_t nb13 = params->src1.nb[3]; + + const size_t nbd1 = params->dst.nb[1]; + const size_t nbd2 = params->dst.nb[2]; + const size_t nbd3 = params->dst.nb[3]; + + // Q2_K super-block holds 256 elements + const int64_t K_blocks = K / QK_K; + + // Broadcasting ratios + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // K-split decision + const int64_t minion_id = hart_id >> 1; /* 0..1023 global */ + const int64_t local_minion = (hart_id >> 1) & 0x1F; /* 0..31 within shire */ + const int is_hart1 = hart_id & 1; + const int64_t rows_per_minion = (M + STRIDE_M_KSPLIT - 1) / STRIDE_M_KSPLIT; + const int64_t k_half = K_blocks / 2; + const int use_ksplit_small_rows = (rows_per_minion <= 2) + && (K_blocks >= KSPLIT_SMALL_ROWS_K_BLOCKS); + /* + * K-split when K is large enough to benefit, and either: + * - few rows (<=4): always safe, proven working + * - more rows (5-8): only if each hart's half fits in one tile, + * otherwise L1 thrashing from 2 harts x 8 rows kills performance + * + * Also allow K-split earlier for the low-M regime (<=2 rows/minion). In + * that case the simple row-striped path leaves half the machine idle, so + * using both harts on each row pays off even for moderate K. + */ + const int use_ksplit = ((K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion <= KSPLIT_MAX_ROWS) + && (rows_per_minion <= 4 || k_half <= TILE_KB)) + || use_ksplit_small_rows; + const int use_ksplit_group = !use_ksplit + && (K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion > 4) + && (rows_per_minion <= KSPLIT_MAX_ROWS); + + if (use_ksplit) { + /* Each hart processes half the K dimension */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + + /* One cache-line-aligned L2SCP slot per minion for exchange */ + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = minion_id; m < M; m += STRIDE_M_KSPLIT) { + const block_q2_K* q_row = (const block_q2_K*)(src0_ptr2 + m * nb01); + + float partial = Q2K_DOT( + q_row + k_start, b_col_base + k_start * QK_K, k_len); + + if (is_hart1) { + *l2scp_slot = partial; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + float other = *l2scp_slot; + et_sem_post(ET_BARRIER_MINION); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, partial + other); + } + } + } + } + } + } else if (use_ksplit_group) { + /* + * Grouped K-split for the 5-8 rows/minion regime. + * + * Both harts process the same 4-row group, each on half of K, and + * exchange 4 partial sums once per group instead of once per row. + */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m_base = minion_id; m_base < M; + m_base += STRIDE_M_KSPLIT * KSPLIT_GROUP_ROWS) { + const int64_t m0 = m_base; + const int64_t m1 = m0 + STRIDE_M_KSPLIT; + const int64_t m2 = m1 + STRIDE_M_KSPLIT; + const int64_t m3 = m2 + STRIDE_M_KSPLIT; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = k_len - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + if (tile_len <= 0) { + break; + } + const float* b_tile = b_col_base + (k_start + kb) * QK_K; + const int64_t row_kb = k_start + kb; + + if (m0 < M) { + s0 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m0 * nb01) + row_kb, + b_tile, tile_len); + } + if (m1 < M) { + s1 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m1 * nb01) + row_kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m2 * nb01) + row_kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m3 * nb01) + row_kb, + b_tile, tile_len); + } + } + + if (is_hart1) { + l2scp_slot[0] = s0; + l2scp_slot[1] = s1; + l2scp_slot[2] = s2; + l2scp_slot[3] = s3; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + const float p0 = l2scp_slot[0]; + const float p1 = l2scp_slot[1]; + const float p2 = l2scp_slot[2]; + const float p3 = l2scp_slot[3]; + et_sem_post(ET_BARRIER_MINION); + + float* c_base = (float*)(dst_ptr2 + n * nbd1); + if (m0 < M) atomic_store_f32((volatile float*)(c_base + m0), s0 + p0); + if (m1 < M) atomic_store_f32((volatile float*)(c_base + m1), s1 + p1); + if (m2 < M) atomic_store_f32((volatile float*)(c_base + m2), s2 + p2); + if (m3 < M) atomic_store_f32((volatile float*)(c_base + m3), s3 + p3); + } + } + } + } + } + } else if (K_blocks > TILE_KB) { + /* + * Tile-outer with scalar row groups: process up to 4 rows per + * hart sharing each B tile before advancing to the next tile. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * 4) { + const int64_t m1 = m0 + STRIDE_M; + const int64_t m2 = m0 + STRIDE_M * 2; + const int64_t m3 = m0 + STRIDE_M * 3; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = K_blocks - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + const float* b_tile = b_col_base + kb * QK_K; + + s0 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m0 * nb01) + kb, + b_tile, tile_len); + if (m1 < M) { + s1 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m1 * nb01) + kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m2 * nb01) + kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m3 * nb01) + kb, + b_tile, tile_len); + } + } + + float* dst_base = (float*)(dst_ptr2 + n * nbd1); + atomic_store_f32((volatile float*)(dst_base + m0), s0); + if (m1 < M) atomic_store_f32((volatile float*)(dst_base + m1), s1); + if (m2 < M) atomic_store_f32((volatile float*)(dst_base + m2), s2); + if (m3 < M) atomic_store_f32((volatile float*)(dst_base + m3), s3); + } + } + } + } + } else { + /* + * Simple path for small K: one row per hart. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q2_K* q_row = (const block_q2_K*)(src0_ptr2 + m * nb01); + + float sum = Q2K_DOT(q_row, b_col_base, K_blocks); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, sum); + } + } + } + } + } + +#ifdef ET_UBERKERNEL + // Publish dst to L3/DRAM for the next uberkernel op. + FENCE; + evict_region_past_l2(params->dst.data, tensor_bytes(¶ms->dst)); + WAIT_CACHEOPS; + FENCE; +#endif + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c new file mode 100644 index 000000000000..b2cc6438f677 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c @@ -0,0 +1,534 @@ +//****************************************************************************** +// MUL_MAT Kernel +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +//****************************************************************************** + +#include +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" +#include "quants.h" +#include "math_fp.h" + +// Q2_K x F32 -> F32 MUL_MAT on the tensor (matrix) engine, TensorFMA32. +// Identical producer/consumer, tiling and tensor-engine loop to +// mul_mat_Q4_K_matrix_engine.c; only the weight dequant differs (Q2_K uses a +// 6-bit quant with a per-16-element int8 scale and no min term). +// Hart 1: dequantize Q2_K weights to FP32 into double-buffered L2 SCP. +// Hart 0: tensor engine compute (FMA, reduce, store). +// +// Two execution paths (selected at runtime by N % TILE_N): +// * REUSE path (N % TILE_N == 0): dequantize each weight K-window ONCE and +// reuse it across ru_n consecutive N-tiles, so the (producer-bound) +// dequant work is cut by ~ru_n. Partial C is round-tripped through an +// L2-SCP scratch between K-windows (the FMA C accumulator is a single fixed +// register-file tile, so multiple output tiles cannot be resident at once). +// * ORIGINAL path (N % TILE_N != 0): one output tile at a time, no reuse. + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +#define TILE_M 16 +#define TILE_N 16 +#define BLOCK_K 32 // one Q2_K group (32 elements) per panel +#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) + +// --- Reuse knobs ---------------------------------------------------------- +// REUSE_MAX caps the L2-SCP C-scratch footprint; the actual reuse factor is +// chosen at runtime as the largest value that still keeps the whole +// machine busy. KWIN is the dequant-cache depth (K-blocks per window). +#ifndef REUSE_MAX +#define REUSE_MAX 15 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per dequant window (cache depth) +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 + +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) + +// Single dequant panel: BLOCK_K k-lines x TILE_M m (FP32) = 32*64 = 2048 bytes, +// [k][m] order: panel[k*TILE_M + m]. +#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t)sizeof(float)) // 2048 + +// L2 SCP layout per minion. The REUSE path needs the larger footprint, so the +// per-minion stride uses it for both paths (mutually exclusive at runtime). +// [0 .. RU_BUF_BYTES) cache buffer 0 (KWIN panels) +// [RU_BUF_BYTES .. 2*..) cache buffer 1 (KWIN panels) +// [RU_CACHE_BYTES .. +R*1024) REUSE_MAX C-scratch tiles (16 rows*64B each) +// ready_ctr, consumed_ctr sync counters +// The ORIGINAL path reuses [0,2048) and [2048,4096) as its two panels and the +// same ready/consumed counters (which sit above the cache region). +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) + +// Software fp16->fp32 (pure integer). The hardware fcvt.ps.f16 returns wrong +// values after the attention block (shared conversion-unit state), which would +// corrupt the weight scales here; software conversion avoids that instruction +// entirely. Only the super-block d is fp16, so cost is negligible. +static inline float __attribute__((always_inline)) me_sw_fp16(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000) << 16; + uint32_t exp = (h >> 10) & 0x1F; + uint32_t mant = h & 0x3FF; + uint32_t f; + if (exp == 0) { + if (mant == 0) { f = sign; } + else { + exp = 127 - 15 + 1; + while ((mant & 0x400) == 0) { mant <<= 1; exp--; } + mant &= 0x3FF; + f = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1F) { + f = sign | 0x7F800000u | (mant << 13); + } else { + f = sign | ((exp + (127 - 15)) << 23) | (mant << 13); + } + float out; __builtin_memcpy(&out, &f, 4); return out; +} + +// EXPERIMENT: use the hardware fcvt.ps.f16 (fp16_to_fp32, math_fp.h) for the +// super-block scale instead of the software me_sw_fp16 above. Hardware fcvt was +// observed to corrupt Q4_K scales after the attention block; trying it here for +// the other K-quants. If it produces garbage, flip ME_FP16 back to me_sw_fp16. +// #define ME_FP16(h) fp16_to_fp32(h) // hardware fcvt: garbage for Q3/Q5/Q6 (fcvt-after-attention bug) +#define ME_FP16(h) me_sw_fp16(h) + +// Dequantize one 32-element Q2_K GROUP of TILE_M weight rows into the FP32 panel, +// written in TenB [k][m] order: panel[k*TILE_M + m]. +// +// A Q2_K super-block packs 256 elements as two 128-element chunks; each 2-bit +// weight q is affine `w = d*(sc&0xF)*q - dmin*(sc>>4)` with a 4-bit scale/min +// pair per 16 elements. kb_group is the global 32-element group index: +// super-block = kb_group/8, group g = kb_group%8, mapping to chunk c = g/4 and +// bit-shift index jj = g%4 (see dequantize_q2_K_block). +static inline void __attribute__((always_inline)) +dequant_q2_K_panel(float *panel, const char *src0_batch, + int64_t mb, int64_t kb_group, int64_t nb1_0) { + const int64_t sb = kb_group >> 3; + const int g = (int) (kb_group & 7); + const int c = g >> 2; // 128-element chunk (0 or 1) + const int jj = g & 3; // sub-group -> shift 2*jj + const int shift = 2 * jj; + const int qoff = c * 32; + const int scoff = c * 8 + 2 * jj; + + for (int j = 0; j < TILE_M; ++j) { + const block_q2_K *blk = + (const block_q2_K *)(src0_batch + (mb + j) * nb1_0) + sb; + const float d = ME_FP16(blk->d); + const float dm = ME_FP16(blk->dmin); + const uint8_t * qs = blk->qs + qoff; + const uint8_t * scl = blk->scales + scoff; + + for (int l = 0; l < 32; ++l) { + const int half = l >> 4; // 0 -> sub A, 1 -> sub B + const uint8_t sc = scl[half]; + const float dl = d * (float) (sc & 0xF); + const float ml = dm * (float) (sc >> 4); + const int8_t q = (int8_t)((qs[l] >> shift) & 3); + panel[l * TILE_M + j] = dl * (float) q - ml; + } + } +} + +// Spill / seed the FP32 C accumulator (16x16 tile in the vector register file, +// row n -> f2n[cols 0..7], f2n+1[cols 8..15]) to/from a 1 KB L2-SCP scratch. +// scratch layout: row n at byte offset n*64. Always moves all 16 rows; rows +// beyond a partial n_cur carry harmless garbage (never stored / recomputed). +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); +} + +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} + +int entry_point(struct ggml_et_binary_params *params, void *env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + // Dimensions (both harts need these for tile assignment) + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + if ((M % TILE_M) != 0) return 0; + if ((K % QK_K) != 0) return 0; + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t k_steps = K / BLOCK_K; // number of 32-element groups + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE; + const int64_t local_tile_idx = local_minion; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + // L2 SCP pointers for this minion. + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // Calculate ru_n to perfectly minimize hardware waves while avoiding Consumer bottleneck. + // The pipeline is perfectly balanced at r=8. Score = waves * max(8, r). + // We find the r that minimizes Score. + int64_t best_r = 1; + int64_t min_score = INT64_MAX; + int64_t max_search_r = REUSE_MAX; + if (max_search_r > n_tiles) max_search_r = n_tiles; + + for (int64_t r = 1; r <= max_search_r; r++) { + int64_t n_groups = (n_tiles + r - 1) / r; + int64_t base_units = m_tiles * n_groups * batch_count; + int64_t waves = (base_units + MACHINE_SLOTS - 1) / MACHINE_SLOTS; + + int64_t penalty = (r > 8) ? r : 8; + int64_t score = waves * penalty; + + if (score < min_score) { + min_score = score; + best_r = r; + } + } + int64_t ru_n = best_r; + + // Reuse pays only when it groups >=2 N-tiles; otherwise the windowing / + // C round-trip is pure overhead, so use the one-tile-at-a-time path. + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: dequant each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t n_groups = (n_tiles + ru_n - 1) / ru_n; + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + const int64_t n_windows = (k_steps + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + dequant_q2_K_panel(cf + i * (SCP_PANEL_SIZE / 4), + src0_batch, mb, kb0 + i, nb1_0); + } + FENCE; + flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } + + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; // first N-tile + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + const int is_last = (kw == n_windows - 1); + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + if (kw > 0) c_seed(cs); + int first = (kw == 0) ? 1 : 0; + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + for (int64_t i = 0; i < kbn; ++i) { + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = (kb0 + i) * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(cf + i * (SCP_PANEL_SIZE / 4) + half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + } + + if (is_last) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } + + // ===================================================================== + // ORIGINAL path: one output tile at a time (N % TILE_N != 0). No reuse. + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { + (float *) et_shire_l2scp_local(scp_base), + (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), + }; + + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); + + dequant_q2_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_panel[buf], BLOCK_K, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + FENCE; + return 0; + } + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t chunk_id = 0; + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + int first = 1; + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + chunk_id++; + scp_wait(ready_ctr, chunk_id); + + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = kb * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + + scp_signal(consumed_ctr, chunk_id); + } + + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K.c new file mode 100644 index 000000000000..41725f41f50c --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K.c @@ -0,0 +1,345 @@ +//****************************************************************************** +// MUL_MAT Kernel (Q3_K weights x F32 activations -> F32) +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +// +// Structure mirrors mul_mat_Q4_K.c. Q3_K also packs 256 elements per +// super-block (16 groups of 16) with per-group int8 scales, so the same +// super-block K-tiling applies; only the per-element dequant differs and is +// delegated to Q3K_DOT() in block_ops.h. +//****************************************************************************** + +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "math_fp.h" +#include "quants.h" +#include "block_ops.h" + +#define STRIDE_M 2048 /* 32 shires x 32 minions x 2 harts */ +#define STRIDE_M_KSPLIT 1024 /* 32 shires x 32 minions (both harts share rows) */ +#define KSPLIT_MIN_K_BLOCKS 32 /* K >= 8192 elements (32 super-blocks) */ +#define KSPLIT_SMALL_ROWS_K_BLOCKS 8 /* K >= 2048 elements for very small M */ +#define KSPLIT_MAX_ROWS 8 /* max rows per minion for K-split */ +#define TILE_KB 32 /* K-tile size in Q3_K super-blocks (8192 elems) */ +#define KSPLIT_GROUP_ROWS 4 + +// Vectorized (8-wide) dot +#define Q3K_DOT(a, b, c) compute_row_dot_q3_K_vec(a, b, c) + +#ifdef ET_UBERKERNEL +static inline size_t tensor_bytes(const struct ggml_tensor* t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} +#endif + +int entry_point(struct ggml_et_binary_params* params, void* env) { + uint64_t hart_id = get_hart_id(); + +#ifdef ET_UBERKERNEL + // Uberkernel coherency: src1 (activations) may be stale in L1/L2; force + // re-read from L3/DRAM. src0 (weights) is read-only, never stale. + evict_region_past_l2(params->src1.data, tensor_bytes(¶ms->src1)); +#endif + + // Matrix dimensions + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + const int64_t ne02 = params->src0.ne[2]; + const int64_t ne03 = params->src0.ne[3]; + const int64_t ne12 = params->src1.ne[2]; + const int64_t ne13 = params->src1.ne[3]; + + // Strides (in bytes) + const size_t nb01 = params->src0.nb[1]; + const size_t nb02 = params->src0.nb[2]; + const size_t nb03 = params->src0.nb[3]; + + const size_t nb11 = params->src1.nb[1]; + const size_t nb12 = params->src1.nb[2]; + const size_t nb13 = params->src1.nb[3]; + + const size_t nbd1 = params->dst.nb[1]; + const size_t nbd2 = params->dst.nb[2]; + const size_t nbd3 = params->dst.nb[3]; + + // Q3_K super-block holds 256 elements + const int64_t K_blocks = K / QK_K; + + // Broadcasting ratios + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // K-split decision + const int64_t minion_id = hart_id >> 1; /* 0..1023 global */ + const int64_t local_minion = (hart_id >> 1) & 0x1F; /* 0..31 within shire */ + const int is_hart1 = hart_id & 1; + const int64_t rows_per_minion = (M + STRIDE_M_KSPLIT - 1) / STRIDE_M_KSPLIT; + const int64_t k_half = K_blocks / 2; + const int use_ksplit_small_rows = (rows_per_minion <= 2) + && (K_blocks >= KSPLIT_SMALL_ROWS_K_BLOCKS); + /* + * K-split when K is large enough to benefit, and either: + * - few rows (<=4): always safe, proven working + * - more rows (5-8): only if each hart's half fits in one tile, + * otherwise L1 thrashing from 2 harts x 8 rows kills performance + * + * Also allow K-split earlier for the low-M regime (<=2 rows/minion). In + * that case the simple row-striped path leaves half the machine idle, so + * using both harts on each row pays off even for moderate K. + */ + const int use_ksplit = ((K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion <= KSPLIT_MAX_ROWS) + && (rows_per_minion <= 4 || k_half <= TILE_KB)) + || use_ksplit_small_rows; + const int use_ksplit_group = !use_ksplit + && (K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion > 4) + && (rows_per_minion <= KSPLIT_MAX_ROWS); + + if (use_ksplit) { + /* Each hart processes half the K dimension */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + + /* One cache-line-aligned L2SCP slot per minion for exchange */ + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = minion_id; m < M; m += STRIDE_M_KSPLIT) { + const block_q3_K* q_row = (const block_q3_K*)(src0_ptr2 + m * nb01); + + float partial = Q3K_DOT( + q_row + k_start, b_col_base + k_start * QK_K, k_len); + + if (is_hart1) { + *l2scp_slot = partial; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + float other = *l2scp_slot; + et_sem_post(ET_BARRIER_MINION); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, partial + other); + } + } + } + } + } + } else if (use_ksplit_group) { + /* + * Grouped K-split for the 5-8 rows/minion regime. + * + * Both harts process the same 4-row group, each on half of K, and + * exchange 4 partial sums once per group instead of once per row. + */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m_base = minion_id; m_base < M; + m_base += STRIDE_M_KSPLIT * KSPLIT_GROUP_ROWS) { + const int64_t m0 = m_base; + const int64_t m1 = m0 + STRIDE_M_KSPLIT; + const int64_t m2 = m1 + STRIDE_M_KSPLIT; + const int64_t m3 = m2 + STRIDE_M_KSPLIT; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = k_len - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + if (tile_len <= 0) { + break; + } + const float* b_tile = b_col_base + (k_start + kb) * QK_K; + const int64_t row_kb = k_start + kb; + + if (m0 < M) { + s0 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m0 * nb01) + row_kb, + b_tile, tile_len); + } + if (m1 < M) { + s1 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m1 * nb01) + row_kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m2 * nb01) + row_kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m3 * nb01) + row_kb, + b_tile, tile_len); + } + } + + if (is_hart1) { + l2scp_slot[0] = s0; + l2scp_slot[1] = s1; + l2scp_slot[2] = s2; + l2scp_slot[3] = s3; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + const float p0 = l2scp_slot[0]; + const float p1 = l2scp_slot[1]; + const float p2 = l2scp_slot[2]; + const float p3 = l2scp_slot[3]; + et_sem_post(ET_BARRIER_MINION); + + float* c_base = (float*)(dst_ptr2 + n * nbd1); + if (m0 < M) atomic_store_f32((volatile float*)(c_base + m0), s0 + p0); + if (m1 < M) atomic_store_f32((volatile float*)(c_base + m1), s1 + p1); + if (m2 < M) atomic_store_f32((volatile float*)(c_base + m2), s2 + p2); + if (m3 < M) atomic_store_f32((volatile float*)(c_base + m3), s3 + p3); + } + } + } + } + } + } else if (K_blocks > TILE_KB) { + /* + * Tile-outer with scalar row groups: process up to 4 rows per + * hart sharing each B tile before advancing to the next tile. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * 4) { + const int64_t m1 = m0 + STRIDE_M; + const int64_t m2 = m0 + STRIDE_M * 2; + const int64_t m3 = m0 + STRIDE_M * 3; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = K_blocks - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + const float* b_tile = b_col_base + kb * QK_K; + + s0 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m0 * nb01) + kb, + b_tile, tile_len); + if (m1 < M) { + s1 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m1 * nb01) + kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m2 * nb01) + kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m3 * nb01) + kb, + b_tile, tile_len); + } + } + + float* dst_base = (float*)(dst_ptr2 + n * nbd1); + atomic_store_f32((volatile float*)(dst_base + m0), s0); + if (m1 < M) atomic_store_f32((volatile float*)(dst_base + m1), s1); + if (m2 < M) atomic_store_f32((volatile float*)(dst_base + m2), s2); + if (m3 < M) atomic_store_f32((volatile float*)(dst_base + m3), s3); + } + } + } + } + } else { + /* + * Simple path for small K: one row per hart. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q3_K* q_row = (const block_q3_K*)(src0_ptr2 + m * nb01); + + float sum = Q3K_DOT(q_row, b_col_base, K_blocks); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, sum); + } + } + } + } + } + +#ifdef ET_UBERKERNEL + // Publish dst to L3/DRAM for the next uberkernel op. + FENCE; + evict_region_past_l2(params->dst.data, tensor_bytes(¶ms->dst)); + WAIT_CACHEOPS; + FENCE; +#endif + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c new file mode 100644 index 000000000000..19bf1b03bb6f --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c @@ -0,0 +1,536 @@ +//****************************************************************************** +// MUL_MAT Kernel +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +//****************************************************************************** + +#include +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" +#include "quants.h" +#include "math_fp.h" + +// Q3_K x F32 -> F32 MUL_MAT on the tensor (matrix) engine, TensorFMA32. +// Identical producer/consumer, tiling and tensor-engine loop to +// mul_mat_Q4_K_matrix_engine.c; only the weight dequant differs (Q3_K uses a +// 6-bit quant with a per-16-element int8 scale and no min term). +// Hart 1: dequantize Q3_K weights to FP32 into double-buffered L2 SCP. +// Hart 0: tensor engine compute (FMA, reduce, store). +// +// Two execution paths (selected at runtime by N % TILE_N): +// * REUSE path (N % TILE_N == 0): dequantize each weight K-window ONCE and +// reuse it across ru_n consecutive N-tiles, so the (producer-bound) +// dequant work is cut by ~ru_n. Partial C is round-tripped through an +// L2-SCP scratch between K-windows (the FMA C accumulator is a single fixed +// register-file tile, so multiple output tiles cannot be resident at once). +// * ORIGINAL path (N % TILE_N != 0): one output tile at a time, no reuse. + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +#define TILE_M 16 +#define TILE_N 16 +#define BLOCK_K 32 // one Q3_K group (32 elements) per panel +#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) + +// --- Reuse knobs ---------------------------------------------------------- +// REUSE_MAX caps the L2-SCP C-scratch footprint; the actual reuse factor is +// chosen at runtime as the largest value that still keeps the whole +// machine busy. KWIN is the dequant-cache depth (K-blocks per window). +#ifndef REUSE_MAX +#define REUSE_MAX 15 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per dequant window (cache depth) +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 + +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) + +// Single dequant panel: BLOCK_K k-lines x TILE_M m (FP32) = 32*64 = 2048 bytes, +// [k][m] order: panel[k*TILE_M + m]. +#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t)sizeof(float)) // 2048 + +// L2 SCP layout per minion. The REUSE path needs the larger footprint, so the +// per-minion stride uses it for both paths (mutually exclusive at runtime). +// [0 .. RU_BUF_BYTES) cache buffer 0 (KWIN panels) +// [RU_BUF_BYTES .. 2*..) cache buffer 1 (KWIN panels) +// [RU_CACHE_BYTES .. +R*1024) REUSE_MAX C-scratch tiles (16 rows*64B each) +// ready_ctr, consumed_ctr sync counters +// The ORIGINAL path reuses [0,2048) and [2048,4096) as its two panels and the +// same ready/consumed counters (which sit above the cache region). +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) + +// Software fp16->fp32 (pure integer). The hardware fcvt.ps.f16 returns wrong +// values after the attention block (shared conversion-unit state), which would +// corrupt the weight scales here; software conversion avoids that instruction +// entirely. Only the super-block d is fp16, so cost is negligible. +static inline float __attribute__((always_inline)) me_sw_fp16(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000) << 16; + uint32_t exp = (h >> 10) & 0x1F; + uint32_t mant = h & 0x3FF; + uint32_t f; + if (exp == 0) { + if (mant == 0) { f = sign; } + else { + exp = 127 - 15 + 1; + while ((mant & 0x400) == 0) { mant <<= 1; exp--; } + mant &= 0x3FF; + f = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1F) { + f = sign | 0x7F800000u | (mant << 13); + } else { + f = sign | ((exp + (127 - 15)) << 23) | (mant << 13); + } + float out; __builtin_memcpy(&out, &f, 4); return out; +} + +// EXPERIMENT: use the hardware fcvt.ps.f16 (fp16_to_fp32, math_fp.h) for the +// super-block scale instead of the software me_sw_fp16 above. Hardware fcvt was +// observed to corrupt Q4_K scales after the attention block; trying it here for +// the other K-quants. If it produces garbage, flip ME_FP16 back to me_sw_fp16. +// #define ME_FP16(h) fp16_to_fp32(h) // hardware fcvt: garbage for Q3/Q5/Q6 (fcvt-after-attention bug) +#define ME_FP16(h) me_sw_fp16(h) + +// Dequantize one 32-element Q3_K GROUP of TILE_M weight rows into the FP32 panel, +// written in TenB [k][m] order: panel[k*TILE_M + m]. +// +// A Q3_K super-block packs 256 elements as two 128-element chunks; each 3-bit +// weight is `w = d*(scale-32)*(q2 - (hbit?0:4))` where q2 is 2 low bits from qs +// and hbit is the high bit from hmask; scales are 16 packed 6-bit values. +// kb_group is the global 32-element group index: super-block = kb_group/8, +// group g = kb_group%8, mapping to chunk c = g/4, bit-shift jj = g%4 and hmask +// bit 1<> 3; + const int g = (int) (kb_group & 7); + const int c = g >> 2; // 128-element chunk (0 or 1) + const int jj = g & 3; // sub-group -> shift 2*jj + const int shift = 2 * jj; + const uint8_t hbit = (uint8_t) (1u << g); + const int qoff = c * 32; + const int scoff = c * 8 + 2 * jj; + + for (int j = 0; j < TILE_M; ++j) { + const block_q3_K *blk = + (const block_q3_K *)(src0_batch + (mb + j) * nb1_0) + sb; + const float d = ME_FP16(blk->d); + const uint8_t * qs = blk->qs + qoff; + const uint8_t * hm = blk->hmask; + int8_t scales[16]; + unpack_q3_K_scales(blk->scales, scales); + + for (int l = 0; l < 32; ++l) { + const int half = l >> 4; + const float dl = d * (float) (scales[scoff + half] - 32); + const int8_t low2 = (int8_t)((qs[l] >> shift) & 3); + const int8_t q = (int8_t)(low2 - ((hm[l] & hbit) ? 0 : 4)); + panel[l * TILE_M + j] = dl * (float) q; + } + } +} + +// Spill / seed the FP32 C accumulator (16x16 tile in the vector register file, +// row n -> f2n[cols 0..7], f2n+1[cols 8..15]) to/from a 1 KB L2-SCP scratch. +// scratch layout: row n at byte offset n*64. Always moves all 16 rows; rows +// beyond a partial n_cur carry harmless garbage (never stored / recomputed). +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); +} + +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} + +int entry_point(struct ggml_et_binary_params *params, void *env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + // Dimensions (both harts need these for tile assignment) + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + if ((M % TILE_M) != 0) return 0; + if ((K % QK_K) != 0) return 0; + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t k_steps = K / BLOCK_K; // number of 32-element groups + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE; + const int64_t local_tile_idx = local_minion; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + // L2 SCP pointers for this minion. + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // Calculate ru_n to perfectly minimize hardware waves while avoiding Consumer bottleneck. + // The pipeline is perfectly balanced at r=8. Score = waves * max(8, r). + // We find the r that minimizes Score. + int64_t best_r = 1; + int64_t min_score = INT64_MAX; + int64_t max_search_r = REUSE_MAX; + if (max_search_r > n_tiles) max_search_r = n_tiles; + + for (int64_t r = 1; r <= max_search_r; r++) { + int64_t n_groups = (n_tiles + r - 1) / r; + int64_t base_units = m_tiles * n_groups * batch_count; + int64_t waves = (base_units + MACHINE_SLOTS - 1) / MACHINE_SLOTS; + + int64_t penalty = (r > 8) ? r : 8; + int64_t score = waves * penalty; + + if (score < min_score) { + min_score = score; + best_r = r; + } + } + int64_t ru_n = best_r; + + // Reuse pays only when it groups >=2 N-tiles; otherwise the windowing / + // C round-trip is pure overhead, so use the one-tile-at-a-time path. + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: dequant each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t n_groups = (n_tiles + ru_n - 1) / ru_n; + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + const int64_t n_windows = (k_steps + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + dequant_q3_K_panel(cf + i * (SCP_PANEL_SIZE / 4), + src0_batch, mb, kb0 + i, nb1_0); + } + FENCE; + flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } + + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; // first N-tile + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + const int is_last = (kw == n_windows - 1); + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + if (kw > 0) c_seed(cs); + int first = (kw == 0) ? 1 : 0; + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + for (int64_t i = 0; i < kbn; ++i) { + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = (kb0 + i) * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(cf + i * (SCP_PANEL_SIZE / 4) + half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + } + + if (is_last) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } + + // ===================================================================== + // ORIGINAL path: one output tile at a time (N % TILE_N != 0). No reuse. + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { + (float *) et_shire_l2scp_local(scp_base), + (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), + }; + + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); + + dequant_q3_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_panel[buf], BLOCK_K, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + FENCE; + return 0; + } + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t chunk_id = 0; + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + int first = 1; + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + chunk_id++; + scp_wait(ready_ctr, chunk_id); + + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = kb * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + + scp_signal(consumed_ctr, chunk_id); + } + + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c index c3af3addd4a1..7a75ea95e9c8 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c @@ -64,25 +64,6 @@ #define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) #define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) -// Signal a counter value to the other hart via L2 SCP. -static inline void __attribute__((always_inline)) -scp_signal(volatile uint32_t *flag, uint32_t value) { - *flag = value; - FENCE; - evict_to_l2((const void *)flag, 1, 64); - WAIT_CACHEOPS; -} - -// Wait for a counter in L2 SCP to reach the expected value. -static inline void __attribute__((always_inline)) -scp_wait(volatile uint32_t *flag, uint32_t expected) { - while (1) { - evict_to_l2((const void *)flag, 1, 64); - WAIT_CACHEOPS; - if (*flag >= expected) return; - } -} - // Dequantize one 32-element Q4_0 block of TILE_M weight rows into the FP32 // panel, written directly in TenB [k][m] order: panel[k*TILE_M + m]. // Low nibble of byte i -> k = i diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K.c index 5ef290d0add1..fd29144bc245 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K.c @@ -5,7 +5,7 @@ // Structure mirrors mul_mat_Q4_0.c. The key difference is the block size: // Q4_K packs 256 elements per super-block (8 groups of 32) with per-group // scales/mins, versus 32 elements per block for Q4_0. The actual dequant + -// dot work is delegated to compute_row_dot_q4_K() in block_ops.h. The K-tiling +// dot work is delegated to Q4K_DOT() in block_ops.h. The K-tiling // and K-split thresholds below are expressed in super-blocks but chosen so the // element-level behaviour matches the Q4_0 kernel (one Q4_K super-block == 8 // Q4_0 blocks, so the block thresholds are divided by 8). @@ -26,9 +26,24 @@ #define TILE_KB 32 /* K-tile size in Q4_K super-blocks (8192 elems, 32KB B data) */ #define KSPLIT_GROUP_ROWS 4 +// Vectorized (8-wide) dot +#define Q4K_DOT(a, b, c) compute_row_dot_q4_K_vec(a, b, c) + +#ifdef ET_UBERKERNEL +static inline size_t tensor_bytes(const struct ggml_tensor* t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} +#endif + int entry_point(struct ggml_et_binary_params* params, void* env) { uint64_t hart_id = get_hart_id(); +#ifdef ET_UBERKERNEL + // Uberkernel coherency: src1 (activations) may be stale in L1/L2 from a + // prior op's write; force re-read from L3/DRAM. src0 (weights) is read-only. + evict_region_past_l2(params->src1.data, tensor_bytes(¶ms->src1)); +#endif + // Matrix dimensions const int64_t K = params->src0.ne[0]; const int64_t M = params->src0.ne[1]; @@ -112,7 +127,7 @@ int entry_point(struct ggml_et_binary_params* params, void* env) { for (int64_t m = minion_id; m < M; m += STRIDE_M_KSPLIT) { const block_q4_K* q_row = (const block_q4_K*)(src0_ptr2 + m * nb01); - float partial = compute_row_dot_q4_K( + float partial = Q4K_DOT( q_row + k_start, b_col_base + k_start * QK_K, k_len); if (is_hart1) { @@ -180,22 +195,22 @@ int entry_point(struct ggml_et_binary_params* params, void* env) { const int64_t row_kb = k_start + kb; if (m0 < M) { - s0 += compute_row_dot_q4_K( + s0 += Q4K_DOT( (const block_q4_K*)(src0_ptr2 + m0 * nb01) + row_kb, b_tile, tile_len); } if (m1 < M) { - s1 += compute_row_dot_q4_K( + s1 += Q4K_DOT( (const block_q4_K*)(src0_ptr2 + m1 * nb01) + row_kb, b_tile, tile_len); } if (m2 < M) { - s2 += compute_row_dot_q4_K( + s2 += Q4K_DOT( (const block_q4_K*)(src0_ptr2 + m2 * nb01) + row_kb, b_tile, tile_len); } if (m3 < M) { - s3 += compute_row_dot_q4_K( + s3 += Q4K_DOT( (const block_q4_K*)(src0_ptr2 + m3 * nb01) + row_kb, b_tile, tile_len); } @@ -261,21 +276,21 @@ int entry_point(struct ggml_et_binary_params* params, void* env) { if (tile_len > TILE_KB) tile_len = TILE_KB; const float* b_tile = b_col_base + kb * QK_K; - s0 += compute_row_dot_q4_K( + s0 += Q4K_DOT( (const block_q4_K*)(src0_ptr2 + m0 * nb01) + kb, b_tile, tile_len); if (m1 < M) { - s1 += compute_row_dot_q4_K( + s1 += Q4K_DOT( (const block_q4_K*)(src0_ptr2 + m1 * nb01) + kb, b_tile, tile_len); } if (m2 < M) { - s2 += compute_row_dot_q4_K( + s2 += Q4K_DOT( (const block_q4_K*)(src0_ptr2 + m2 * nb01) + kb, b_tile, tile_len); } if (m3 < M) { - s3 += compute_row_dot_q4_K( + s3 += Q4K_DOT( (const block_q4_K*)(src0_ptr2 + m3 * nb01) + kb, b_tile, tile_len); } @@ -312,7 +327,7 @@ int entry_point(struct ggml_et_binary_params* params, void* env) { for (int64_t m = hart_id; m < M; m += STRIDE_M) { const block_q4_K* q_row = (const block_q4_K*)(src0_ptr2 + m * nb01); - float sum = compute_row_dot_q4_K(q_row, b_col_base, K_blocks); + float sum = Q4K_DOT(q_row, b_col_base, K_blocks); float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); atomic_store_f32((volatile float*)dst_entry, sum); @@ -322,5 +337,12 @@ int entry_point(struct ggml_et_binary_params* params, void* env) { } } +#ifdef ET_UBERKERNEL + // Publish dst to L3/DRAM so the next uberkernel op reads fresh data. + FENCE; + evict_region_past_l2(params->dst.data, tensor_bytes(¶ms->dst)); + WAIT_CACHEOPS; + FENCE; +#endif return 0; } diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c index f3abdeba54c8..f8f5eb4d3350 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c @@ -43,6 +43,17 @@ #ifndef KWIN #define KWIN 16 // K-blocks per dequant window (cache depth) #endif +// BAL is the producer/consumer balance point: reuse factors r<=BAL are treated +// as "free" (dequant is the bottleneck), r>BAL is penalized because the +// consumer (+ C round-trip) becomes the bottleneck. TIE_PREFER_LARGE selects +// the LARGEST r among equal scores (max reuse / least dequant) instead of the +// smallest. +#ifndef BAL +#define BAL 8 +#endif +#ifndef TIE_PREFER_LARGE +#define TIE_PREFER_LARGE 0 +#endif #define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 @@ -71,25 +82,6 @@ #define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) #define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) -// Signal a counter value to the other hart via L2 SCP. -static inline void __attribute__((always_inline)) -scp_signal(volatile uint32_t *flag, uint32_t value) { - *flag = value; - FENCE; - evict_to_l2((const void *)flag, 1, 64); - WAIT_CACHEOPS; -} - -// Wait for a counter in L2 SCP to reach the expected value. -static inline void __attribute__((always_inline)) -scp_wait(volatile uint32_t *flag, uint32_t expected) { - while (1) { - evict_to_l2((const void *)flag, 1, 64); - WAIT_CACHEOPS; - if (*flag >= expected) return; - } -} - // Software fp16->fp32 (pure integer). The hardware fcvt.ps.f16 returns wrong // values after the attention block (shared conversion-unit state), which would // corrupt the Q4_K weight scales here; software conversion avoids that @@ -306,14 +298,21 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { int64_t n_groups = (n_tiles + r - 1) / r; int64_t base_units = m_tiles * n_groups * batch_count; int64_t waves = (base_units + MACHINE_SLOTS - 1) / MACHINE_SLOTS; - - int64_t penalty = (r > 8) ? r : 8; + + int64_t penalty = (r > BAL) ? r : BAL; int64_t score = waves * penalty; - + +#if TIE_PREFER_LARGE + if (score <= min_score) { // largest r among equal scores = max reuse + min_score = score; + best_r = r; + } +#else if (score < min_score) { min_score = score; best_r = r; } +#endif } int64_t ru_n = best_r; diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K.c new file mode 100644 index 000000000000..626a0a10c8f0 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K.c @@ -0,0 +1,345 @@ +//****************************************************************************** +// MUL_MAT Kernel (Q5_K weights x F32 activations -> F32) +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +// +// Structure mirrors mul_mat_Q4_K.c. Q5_K also packs 256 elements per +// super-block (16 groups of 16) with per-group int8 scales, so the same +// super-block K-tiling applies; only the per-element dequant differs and is +// delegated to Q5K_DOT() in block_ops.h. +//****************************************************************************** + +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "math_fp.h" +#include "quants.h" +#include "block_ops.h" + +#define STRIDE_M 2048 /* 32 shires x 32 minions x 2 harts */ +#define STRIDE_M_KSPLIT 1024 /* 32 shires x 32 minions (both harts share rows) */ +#define KSPLIT_MIN_K_BLOCKS 32 /* K >= 8192 elements (32 super-blocks) */ +#define KSPLIT_SMALL_ROWS_K_BLOCKS 8 /* K >= 2048 elements for very small M */ +#define KSPLIT_MAX_ROWS 8 /* max rows per minion for K-split */ +#define TILE_KB 32 /* K-tile size in Q5_K super-blocks (8192 elems) */ +#define KSPLIT_GROUP_ROWS 4 + +// Vectorized (8-wide) dot +#define Q5K_DOT(a, b, c) compute_row_dot_q5_K_vec(a, b, c) + +#ifdef ET_UBERKERNEL +static inline size_t tensor_bytes(const struct ggml_tensor* t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} +#endif + +int entry_point(struct ggml_et_binary_params* params, void* env) { + uint64_t hart_id = get_hart_id(); + +#ifdef ET_UBERKERNEL + // Uberkernel coherency: src1 (activations) may be stale in L1/L2; force + // re-read from L3/DRAM. src0 (weights) is read-only, never stale. + evict_region_past_l2(params->src1.data, tensor_bytes(¶ms->src1)); +#endif + + // Matrix dimensions + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + const int64_t ne02 = params->src0.ne[2]; + const int64_t ne03 = params->src0.ne[3]; + const int64_t ne12 = params->src1.ne[2]; + const int64_t ne13 = params->src1.ne[3]; + + // Strides (in bytes) + const size_t nb01 = params->src0.nb[1]; + const size_t nb02 = params->src0.nb[2]; + const size_t nb03 = params->src0.nb[3]; + + const size_t nb11 = params->src1.nb[1]; + const size_t nb12 = params->src1.nb[2]; + const size_t nb13 = params->src1.nb[3]; + + const size_t nbd1 = params->dst.nb[1]; + const size_t nbd2 = params->dst.nb[2]; + const size_t nbd3 = params->dst.nb[3]; + + // Q5_K super-block holds 256 elements + const int64_t K_blocks = K / QK_K; + + // Broadcasting ratios + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // K-split decision + const int64_t minion_id = hart_id >> 1; /* 0..1023 global */ + const int64_t local_minion = (hart_id >> 1) & 0x1F; /* 0..31 within shire */ + const int is_hart1 = hart_id & 1; + const int64_t rows_per_minion = (M + STRIDE_M_KSPLIT - 1) / STRIDE_M_KSPLIT; + const int64_t k_half = K_blocks / 2; + const int use_ksplit_small_rows = (rows_per_minion <= 2) + && (K_blocks >= KSPLIT_SMALL_ROWS_K_BLOCKS); + /* + * K-split when K is large enough to benefit, and either: + * - few rows (<=4): always safe, proven working + * - more rows (5-8): only if each hart's half fits in one tile, + * otherwise L1 thrashing from 2 harts x 8 rows kills performance + * + * Also allow K-split earlier for the low-M regime (<=2 rows/minion). In + * that case the simple row-striped path leaves half the machine idle, so + * using both harts on each row pays off even for moderate K. + */ + const int use_ksplit = ((K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion <= KSPLIT_MAX_ROWS) + && (rows_per_minion <= 4 || k_half <= TILE_KB)) + || use_ksplit_small_rows; + const int use_ksplit_group = !use_ksplit + && (K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion > 4) + && (rows_per_minion <= KSPLIT_MAX_ROWS); + + if (use_ksplit) { + /* Each hart processes half the K dimension */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + + /* One cache-line-aligned L2SCP slot per minion for exchange */ + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = minion_id; m < M; m += STRIDE_M_KSPLIT) { + const block_q5_K* q_row = (const block_q5_K*)(src0_ptr2 + m * nb01); + + float partial = Q5K_DOT( + q_row + k_start, b_col_base + k_start * QK_K, k_len); + + if (is_hart1) { + *l2scp_slot = partial; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + float other = *l2scp_slot; + et_sem_post(ET_BARRIER_MINION); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, partial + other); + } + } + } + } + } + } else if (use_ksplit_group) { + /* + * Grouped K-split for the 5-8 rows/minion regime. + * + * Both harts process the same 4-row group, each on half of K, and + * exchange 4 partial sums once per group instead of once per row. + */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m_base = minion_id; m_base < M; + m_base += STRIDE_M_KSPLIT * KSPLIT_GROUP_ROWS) { + const int64_t m0 = m_base; + const int64_t m1 = m0 + STRIDE_M_KSPLIT; + const int64_t m2 = m1 + STRIDE_M_KSPLIT; + const int64_t m3 = m2 + STRIDE_M_KSPLIT; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = k_len - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + if (tile_len <= 0) { + break; + } + const float* b_tile = b_col_base + (k_start + kb) * QK_K; + const int64_t row_kb = k_start + kb; + + if (m0 < M) { + s0 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m0 * nb01) + row_kb, + b_tile, tile_len); + } + if (m1 < M) { + s1 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m1 * nb01) + row_kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m2 * nb01) + row_kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m3 * nb01) + row_kb, + b_tile, tile_len); + } + } + + if (is_hart1) { + l2scp_slot[0] = s0; + l2scp_slot[1] = s1; + l2scp_slot[2] = s2; + l2scp_slot[3] = s3; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + const float p0 = l2scp_slot[0]; + const float p1 = l2scp_slot[1]; + const float p2 = l2scp_slot[2]; + const float p3 = l2scp_slot[3]; + et_sem_post(ET_BARRIER_MINION); + + float* c_base = (float*)(dst_ptr2 + n * nbd1); + if (m0 < M) atomic_store_f32((volatile float*)(c_base + m0), s0 + p0); + if (m1 < M) atomic_store_f32((volatile float*)(c_base + m1), s1 + p1); + if (m2 < M) atomic_store_f32((volatile float*)(c_base + m2), s2 + p2); + if (m3 < M) atomic_store_f32((volatile float*)(c_base + m3), s3 + p3); + } + } + } + } + } + } else if (K_blocks > TILE_KB) { + /* + * Tile-outer with scalar row groups: process up to 4 rows per + * hart sharing each B tile before advancing to the next tile. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * 4) { + const int64_t m1 = m0 + STRIDE_M; + const int64_t m2 = m0 + STRIDE_M * 2; + const int64_t m3 = m0 + STRIDE_M * 3; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = K_blocks - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + const float* b_tile = b_col_base + kb * QK_K; + + s0 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m0 * nb01) + kb, + b_tile, tile_len); + if (m1 < M) { + s1 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m1 * nb01) + kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m2 * nb01) + kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m3 * nb01) + kb, + b_tile, tile_len); + } + } + + float* dst_base = (float*)(dst_ptr2 + n * nbd1); + atomic_store_f32((volatile float*)(dst_base + m0), s0); + if (m1 < M) atomic_store_f32((volatile float*)(dst_base + m1), s1); + if (m2 < M) atomic_store_f32((volatile float*)(dst_base + m2), s2); + if (m3 < M) atomic_store_f32((volatile float*)(dst_base + m3), s3); + } + } + } + } + } else { + /* + * Simple path for small K: one row per hart. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q5_K* q_row = (const block_q5_K*)(src0_ptr2 + m * nb01); + + float sum = Q5K_DOT(q_row, b_col_base, K_blocks); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, sum); + } + } + } + } + } + +#ifdef ET_UBERKERNEL + // Publish dst to L3/DRAM for the next uberkernel op. + FENCE; + evict_region_past_l2(params->dst.data, tensor_bytes(¶ms->dst)); + WAIT_CACHEOPS; + FENCE; +#endif + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c new file mode 100644 index 000000000000..e78e4102bef2 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c @@ -0,0 +1,535 @@ +//****************************************************************************** +// MUL_MAT Kernel +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +//****************************************************************************** + +#include +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" +#include "quants.h" +#include "math_fp.h" + +// Q5_K x F32 -> F32 MUL_MAT on the tensor (matrix) engine, TensorFMA32. +// Identical producer/consumer, tiling and tensor-engine loop to +// mul_mat_Q4_K_matrix_engine.c; only the weight dequant differs (Q5_K uses a +// 6-bit quant with a per-16-element int8 scale and no min term). +// Hart 1: dequantize Q5_K weights to FP32 into double-buffered L2 SCP. +// Hart 0: tensor engine compute (FMA, reduce, store). +// +// Two execution paths (selected at runtime by N % TILE_N): +// * REUSE path (N % TILE_N == 0): dequantize each weight K-window ONCE and +// reuse it across ru_n consecutive N-tiles, so the (producer-bound) +// dequant work is cut by ~ru_n. Partial C is round-tripped through an +// L2-SCP scratch between K-windows (the FMA C accumulator is a single fixed +// register-file tile, so multiple output tiles cannot be resident at once). +// * ORIGINAL path (N % TILE_N != 0): one output tile at a time, no reuse. + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +#define TILE_M 16 +#define TILE_N 16 +#define BLOCK_K 32 // one Q5_K group (32 elements) per panel +#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) + +// --- Reuse knobs ---------------------------------------------------------- +// REUSE_MAX caps the L2-SCP C-scratch footprint; the actual reuse factor is +// chosen at runtime as the largest value that still keeps the whole +// machine busy. KWIN is the dequant-cache depth (K-blocks per window). +#ifndef REUSE_MAX +#define REUSE_MAX 15 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per dequant window (cache depth) +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 + +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) + +// Single dequant panel: BLOCK_K k-lines x TILE_M m (FP32) = 32*64 = 2048 bytes, +// [k][m] order: panel[k*TILE_M + m]. +#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t)sizeof(float)) // 2048 + +// L2 SCP layout per minion. The REUSE path needs the larger footprint, so the +// per-minion stride uses it for both paths (mutually exclusive at runtime). +// [0 .. RU_BUF_BYTES) cache buffer 0 (KWIN panels) +// [RU_BUF_BYTES .. 2*..) cache buffer 1 (KWIN panels) +// [RU_CACHE_BYTES .. +R*1024) REUSE_MAX C-scratch tiles (16 rows*64B each) +// ready_ctr, consumed_ctr sync counters +// The ORIGINAL path reuses [0,2048) and [2048,4096) as its two panels and the +// same ready/consumed counters (which sit above the cache region). +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) + +// Software fp16->fp32 (pure integer). The hardware fcvt.ps.f16 returns wrong +// values after the attention block (shared conversion-unit state), which would +// corrupt the weight scales here; software conversion avoids that instruction +// entirely. Only the super-block d is fp16, so cost is negligible. +static inline float __attribute__((always_inline)) me_sw_fp16(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000) << 16; + uint32_t exp = (h >> 10) & 0x1F; + uint32_t mant = h & 0x3FF; + uint32_t f; + if (exp == 0) { + if (mant == 0) { f = sign; } + else { + exp = 127 - 15 + 1; + while ((mant & 0x400) == 0) { mant <<= 1; exp--; } + mant &= 0x3FF; + f = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1F) { + f = sign | 0x7F800000u | (mant << 13); + } else { + f = sign | ((exp + (127 - 15)) << 23) | (mant << 13); + } + float out; __builtin_memcpy(&out, &f, 4); return out; +} + +// EXPERIMENT: use the hardware fcvt.ps.f16 (fp16_to_fp32, math_fp.h) for the +// super-block scale instead of the software me_sw_fp16 above. Hardware fcvt was +// observed to corrupt Q4_K scales after the attention block; trying it here for +// the other K-quants. If it produces garbage, flip ME_FP16 back to me_sw_fp16. +// #define ME_FP16(h) fp16_to_fp32(h) // hardware fcvt: garbage for Q3/Q5/Q6 (fcvt-after-attention bug) +#define ME_FP16(h) me_sw_fp16(h) + +// Dequantize one 32-element Q5_K GROUP of TILE_M weight rows into the FP32 panel, +// written in TenB [k][m] order: panel[k*TILE_M + m]. +// +// A Q5_K super-block has the same affine layout as Q4_K (8 groups of 32) with an +// extra high bit per weight drawn from qh: +// w = d*sc*((nibble) + (qh_bit ? 16 : 0)) - dmin*m +// kb_group is the global 32-element group index: super-block = kb_group/8, +// group g = kb_group%8 -> qs pair p = g/2, low/high nibble ab = g%1, qh bit +// 1<<(2p+ab) (see dequantize_q5_K_block). +static inline void __attribute__((always_inline)) +dequant_q5_K_panel(float *panel, const char *src0_batch, + int64_t mb, int64_t kb_group, int64_t nb1_0) { + const int64_t sb = kb_group >> 3; + const int g = (int) (kb_group & 7); + const int p = g >> 1; // qs pair (0..3), 32-byte block + const int ab = g & 1; // 0 low nibble, 1 high nibble + const int qloff = p * 32; + const uint8_t ubit = (uint8_t) (1u << (2 * p + ab)); + + for (int j = 0; j < TILE_M; ++j) { + const block_q5_K *blk = + (const block_q5_K *)(src0_batch + (mb + j) * nb1_0) + sb; + const float d = ME_FP16(blk->d); + const float dm = ME_FP16(blk->dmin); + const uint8_t * ql = blk->qs + qloff; + const uint8_t * qh = blk->qh; + uint8_t sc, mm; + get_scale_min_k4(g, blk->scales, &sc, &mm); + const float dl = d * (float) sc; + const float ml = dm * (float) mm; + + for (int l = 0; l < 32; ++l) { + const uint8_t nib = ab ? (ql[l] >> 4) : (ql[l] & 0xF); + const int hi5 = (qh[l] & ubit) ? 16 : 0; + panel[l * TILE_M + j] = dl * (float) (nib + hi5) - ml; + } + } +} + +// Spill / seed the FP32 C accumulator (16x16 tile in the vector register file, +// row n -> f2n[cols 0..7], f2n+1[cols 8..15]) to/from a 1 KB L2-SCP scratch. +// scratch layout: row n at byte offset n*64. Always moves all 16 rows; rows +// beyond a partial n_cur carry harmless garbage (never stored / recomputed). +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); +} + +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} + +int entry_point(struct ggml_et_binary_params *params, void *env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + // Dimensions (both harts need these for tile assignment) + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + if ((M % TILE_M) != 0) return 0; + if ((K % QK_K) != 0) return 0; + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t k_steps = K / BLOCK_K; // number of 32-element groups + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE; + const int64_t local_tile_idx = local_minion; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + // L2 SCP pointers for this minion. + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // Calculate ru_n to perfectly minimize hardware waves while avoiding Consumer bottleneck. + // The pipeline is perfectly balanced at r=8. Score = waves * max(8, r). + // We find the r that minimizes Score. + int64_t best_r = 1; + int64_t min_score = INT64_MAX; + int64_t max_search_r = REUSE_MAX; + if (max_search_r > n_tiles) max_search_r = n_tiles; + + for (int64_t r = 1; r <= max_search_r; r++) { + int64_t n_groups = (n_tiles + r - 1) / r; + int64_t base_units = m_tiles * n_groups * batch_count; + int64_t waves = (base_units + MACHINE_SLOTS - 1) / MACHINE_SLOTS; + + int64_t penalty = (r > 8) ? r : 8; + int64_t score = waves * penalty; + + if (score < min_score) { + min_score = score; + best_r = r; + } + } + int64_t ru_n = best_r; + + // Reuse pays only when it groups >=2 N-tiles; otherwise the windowing / + // C round-trip is pure overhead, so use the one-tile-at-a-time path. + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: dequant each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t n_groups = (n_tiles + ru_n - 1) / ru_n; + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + const int64_t n_windows = (k_steps + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + dequant_q5_K_panel(cf + i * (SCP_PANEL_SIZE / 4), + src0_batch, mb, kb0 + i, nb1_0); + } + FENCE; + flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } + + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; // first N-tile + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + const int is_last = (kw == n_windows - 1); + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + if (kw > 0) c_seed(cs); + int first = (kw == 0) ? 1 : 0; + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + for (int64_t i = 0; i < kbn; ++i) { + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = (kb0 + i) * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(cf + i * (SCP_PANEL_SIZE / 4) + half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + } + + if (is_last) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } + + // ===================================================================== + // ORIGINAL path: one output tile at a time (N % TILE_N != 0). No reuse. + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { + (float *) et_shire_l2scp_local(scp_base), + (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), + }; + + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); + + dequant_q5_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_panel[buf], BLOCK_K, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + FENCE; + return 0; + } + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t chunk_id = 0; + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + int first = 1; + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + chunk_id++; + scp_wait(ready_ctr, chunk_id); + + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = kb * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + + scp_signal(consumed_ctr, chunk_id); + } + + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K.c new file mode 100644 index 000000000000..0a0dbb48e463 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K.c @@ -0,0 +1,347 @@ +//****************************************************************************** +// MUL_MAT Kernel (Q6_K weights x F32 activations -> F32) +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +// +// Structure mirrors mul_mat_Q4_K.c. Q6_K also packs 256 elements per +// super-block (16 groups of 16) with per-group int8 scales, so the same +// super-block K-tiling applies; only the per-element dequant differs and is +// delegated to Q6K_DOT() in block_ops.h. +//****************************************************************************** + +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "math_fp.h" +#include "quants.h" +#include "block_ops.h" + +#define STRIDE_M 2048 /* 32 shires x 32 minions x 2 harts */ +#define STRIDE_M_KSPLIT 1024 /* 32 shires x 32 minions (both harts share rows) */ +#define KSPLIT_MIN_K_BLOCKS 32 /* K >= 8192 elements (32 super-blocks) */ +#define KSPLIT_SMALL_ROWS_K_BLOCKS 8 /* K >= 2048 elements for very small M */ +#define KSPLIT_MAX_ROWS 8 /* max rows per minion for K-split */ +#define TILE_KB 32 /* K-tile size in Q6_K super-blocks (8192 elems) */ +#define KSPLIT_GROUP_ROWS 4 + +// Vectorized (8-wide) dot +// (Gather-reduction and multi-accumulator variants were tried and dropped - +// both perf-neutral; the generation dot is memory/overhead bound, not compute.) +#define Q6K_DOT(a, b, c) compute_row_dot_q6_K_vec(a, b, c) + +#ifdef ET_UBERKERNEL +static inline size_t tensor_bytes(const struct ggml_tensor* t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} +#endif + +int entry_point(struct ggml_et_binary_params* params, void* env) { + uint64_t hart_id = get_hart_id(); + +#ifdef ET_UBERKERNEL + // Uberkernel coherency: src1 (activations) may be stale in L1/L2; force + // re-read from L3/DRAM. src0 (weights) is read-only, never stale. + evict_region_past_l2(params->src1.data, tensor_bytes(¶ms->src1)); +#endif + + // Matrix dimensions + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + const int64_t ne02 = params->src0.ne[2]; + const int64_t ne03 = params->src0.ne[3]; + const int64_t ne12 = params->src1.ne[2]; + const int64_t ne13 = params->src1.ne[3]; + + // Strides (in bytes) + const size_t nb01 = params->src0.nb[1]; + const size_t nb02 = params->src0.nb[2]; + const size_t nb03 = params->src0.nb[3]; + + const size_t nb11 = params->src1.nb[1]; + const size_t nb12 = params->src1.nb[2]; + const size_t nb13 = params->src1.nb[3]; + + const size_t nbd1 = params->dst.nb[1]; + const size_t nbd2 = params->dst.nb[2]; + const size_t nbd3 = params->dst.nb[3]; + + // Q6_K super-block holds 256 elements + const int64_t K_blocks = K / QK_K; + + // Broadcasting ratios + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // K-split decision + const int64_t minion_id = hart_id >> 1; /* 0..1023 global */ + const int64_t local_minion = (hart_id >> 1) & 0x1F; /* 0..31 within shire */ + const int is_hart1 = hart_id & 1; + const int64_t rows_per_minion = (M + STRIDE_M_KSPLIT - 1) / STRIDE_M_KSPLIT; + const int64_t k_half = K_blocks / 2; + const int use_ksplit_small_rows = (rows_per_minion <= 2) + && (K_blocks >= KSPLIT_SMALL_ROWS_K_BLOCKS); + /* + * K-split when K is large enough to benefit, and either: + * - few rows (<=4): always safe, proven working + * - more rows (5-8): only if each hart's half fits in one tile, + * otherwise L1 thrashing from 2 harts x 8 rows kills performance + * + * Also allow K-split earlier for the low-M regime (<=2 rows/minion). In + * that case the simple row-striped path leaves half the machine idle, so + * using both harts on each row pays off even for moderate K. + */ + const int use_ksplit = ((K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion <= KSPLIT_MAX_ROWS) + && (rows_per_minion <= 4 || k_half <= TILE_KB)) + || use_ksplit_small_rows; + const int use_ksplit_group = !use_ksplit + && (K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion > 4) + && (rows_per_minion <= KSPLIT_MAX_ROWS); + + if (use_ksplit) { + /* Each hart processes half the K dimension */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + + /* One cache-line-aligned L2SCP slot per minion for exchange */ + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = minion_id; m < M; m += STRIDE_M_KSPLIT) { + const block_q6_K* q_row = (const block_q6_K*)(src0_ptr2 + m * nb01); + + float partial = Q6K_DOT( + q_row + k_start, b_col_base + k_start * QK_K, k_len); + + if (is_hart1) { + *l2scp_slot = partial; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + float other = *l2scp_slot; + et_sem_post(ET_BARRIER_MINION); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, partial + other); + } + } + } + } + } + } else if (use_ksplit_group) { + /* + * Grouped K-split for the 5-8 rows/minion regime. + * + * Both harts process the same 4-row group, each on half of K, and + * exchange 4 partial sums once per group instead of once per row. + */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m_base = minion_id; m_base < M; + m_base += STRIDE_M_KSPLIT * KSPLIT_GROUP_ROWS) { + const int64_t m0 = m_base; + const int64_t m1 = m0 + STRIDE_M_KSPLIT; + const int64_t m2 = m1 + STRIDE_M_KSPLIT; + const int64_t m3 = m2 + STRIDE_M_KSPLIT; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = k_len - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + if (tile_len <= 0) { + break; + } + const float* b_tile = b_col_base + (k_start + kb) * QK_K; + const int64_t row_kb = k_start + kb; + + if (m0 < M) { + s0 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m0 * nb01) + row_kb, + b_tile, tile_len); + } + if (m1 < M) { + s1 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m1 * nb01) + row_kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m2 * nb01) + row_kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m3 * nb01) + row_kb, + b_tile, tile_len); + } + } + + if (is_hart1) { + l2scp_slot[0] = s0; + l2scp_slot[1] = s1; + l2scp_slot[2] = s2; + l2scp_slot[3] = s3; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + const float p0 = l2scp_slot[0]; + const float p1 = l2scp_slot[1]; + const float p2 = l2scp_slot[2]; + const float p3 = l2scp_slot[3]; + et_sem_post(ET_BARRIER_MINION); + + float* c_base = (float*)(dst_ptr2 + n * nbd1); + if (m0 < M) atomic_store_f32((volatile float*)(c_base + m0), s0 + p0); + if (m1 < M) atomic_store_f32((volatile float*)(c_base + m1), s1 + p1); + if (m2 < M) atomic_store_f32((volatile float*)(c_base + m2), s2 + p2); + if (m3 < M) atomic_store_f32((volatile float*)(c_base + m3), s3 + p3); + } + } + } + } + } + } else if (K_blocks > TILE_KB) { + /* + * Tile-outer with scalar row groups: process up to 4 rows per + * hart sharing each B tile before advancing to the next tile. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * 4) { + const int64_t m1 = m0 + STRIDE_M; + const int64_t m2 = m0 + STRIDE_M * 2; + const int64_t m3 = m0 + STRIDE_M * 3; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = K_blocks - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + const float* b_tile = b_col_base + kb * QK_K; + + s0 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m0 * nb01) + kb, + b_tile, tile_len); + if (m1 < M) { + s1 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m1 * nb01) + kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m2 * nb01) + kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m3 * nb01) + kb, + b_tile, tile_len); + } + } + + float* dst_base = (float*)(dst_ptr2 + n * nbd1); + atomic_store_f32((volatile float*)(dst_base + m0), s0); + if (m1 < M) atomic_store_f32((volatile float*)(dst_base + m1), s1); + if (m2 < M) atomic_store_f32((volatile float*)(dst_base + m2), s2); + if (m3 < M) atomic_store_f32((volatile float*)(dst_base + m3), s3); + } + } + } + } + } else { + /* + * Simple path for small K: one row per hart. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q6_K* q_row = (const block_q6_K*)(src0_ptr2 + m * nb01); + + float sum = Q6K_DOT(q_row, b_col_base, K_blocks); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, sum); + } + } + } + } + } + +#ifdef ET_UBERKERNEL + // Publish dst to L3/DRAM for the next uberkernel op. + FENCE; + evict_region_past_l2(params->dst.data, tensor_bytes(¶ms->dst)); + WAIT_CACHEOPS; + FENCE; +#endif + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c new file mode 100644 index 000000000000..1345df09453c --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c @@ -0,0 +1,538 @@ +//****************************************************************************** +// MUL_MAT Kernel +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +//****************************************************************************** + +#include +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" +#include "quants.h" +#include "math_fp.h" + +// Q6_K x F32 -> F32 MUL_MAT on the tensor (matrix) engine, TensorFMA32. +// Identical producer/consumer, tiling and tensor-engine loop to +// mul_mat_Q4_K_matrix_engine.c; only the weight dequant differs (Q6_K uses a +// 6-bit quant with a per-16-element int8 scale and no min term). +// Hart 1: dequantize Q6_K weights to FP32 into double-buffered L2 SCP. +// Hart 0: tensor engine compute (FMA, reduce, store). +// +// Two execution paths (selected at runtime by N % TILE_N): +// * REUSE path (N % TILE_N == 0): dequantize each weight K-window ONCE and +// reuse it across ru_n consecutive N-tiles, so the (producer-bound) +// dequant work is cut by ~ru_n. Partial C is round-tripped through an +// L2-SCP scratch between K-windows (the FMA C accumulator is a single fixed +// register-file tile, so multiple output tiles cannot be resident at once). +// * ORIGINAL path (N % TILE_N != 0): one output tile at a time, no reuse. + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +#define TILE_M 16 +#define TILE_N 16 +#define BLOCK_K 32 // one Q6_K group (32 elements) per panel +#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) + +// --- Reuse knobs ---------------------------------------------------------- +// REUSE_MAX caps the L2-SCP C-scratch footprint; the actual reuse factor is +// chosen at runtime as the largest value that still keeps the whole +// machine busy. KWIN is the dequant-cache depth (K-blocks per window). +#ifndef REUSE_MAX +#define REUSE_MAX 15 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per dequant window (cache depth) +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 + +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) + +// Single dequant panel: BLOCK_K k-lines x TILE_M m (FP32) = 32*64 = 2048 bytes, +// [k][m] order: panel[k*TILE_M + m]. +#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t)sizeof(float)) // 2048 + +// L2 SCP layout per minion. The REUSE path needs the larger footprint, so the +// per-minion stride uses it for both paths (mutually exclusive at runtime). +// [0 .. RU_BUF_BYTES) cache buffer 0 (KWIN panels) +// [RU_BUF_BYTES .. 2*..) cache buffer 1 (KWIN panels) +// [RU_CACHE_BYTES .. +R*1024) REUSE_MAX C-scratch tiles (16 rows*64B each) +// ready_ctr, consumed_ctr sync counters +// The ORIGINAL path reuses [0,2048) and [2048,4096) as its two panels and the +// same ready/consumed counters (which sit above the cache region). +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) + + + +// Software fp16->fp32 (pure integer). The hardware fcvt.ps.f16 returns wrong +// values after the attention block (shared conversion-unit state), which would +// corrupt the weight scales here; software conversion avoids that instruction +// entirely. Only the super-block d is fp16, so cost is negligible. +static inline float __attribute__((always_inline)) me_sw_fp16(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000) << 16; + uint32_t exp = (h >> 10) & 0x1F; + uint32_t mant = h & 0x3FF; + uint32_t f; + if (exp == 0) { + if (mant == 0) { f = sign; } + else { + exp = 127 - 15 + 1; + while ((mant & 0x400) == 0) { mant <<= 1; exp--; } + mant &= 0x3FF; + f = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1F) { + f = sign | 0x7F800000u | (mant << 13); + } else { + f = sign | ((exp + (127 - 15)) << 23) | (mant << 13); + } + float out; __builtin_memcpy(&out, &f, 4); return out; +} + +// EXPERIMENT: use the hardware fcvt.ps.f16 (fp16_to_fp32, math_fp.h) for the +// super-block scale instead of the software me_sw_fp16 above. Hardware fcvt was +// observed to corrupt Q4_K scales after the attention block; trying it here for +// the other K-quants. If it produces garbage, flip ME_FP16 back to me_sw_fp16. +// #define ME_FP16(h) fp16_to_fp32(h) // hardware fcvt: garbage for Q3/Q5/Q6 (fcvt-after-attention bug) +#define ME_FP16(h) me_sw_fp16(h) + +// Dequantize one 32-element Q6_K GROUP of TILE_M weight rows into the FP32 panel, +// written in TenB [k][m] order: panel[k*TILE_M + m]. +// +// A Q6_K super-block packs 256 elements as two 128-element chunks of four +// 32-element groups; each 6-bit weight is (ql nibble | qh 2-bit) - 32, scaled +// by a per-16-element int8 scale and the super-block fp16 d: +// w = d * scale * ((nibble | (bits << 4)) - 32) +// kb_group is the global group index: super-block = kb_group/8, group g = kb_group%8. +// Group g maps to chunk c = g/4 and sub-group sub = g%4 (see dequantize_q6_K_block). +static inline void __attribute__((always_inline)) +dequant_q6_K_panel(float *panel, const char *src0_batch, + int64_t mb, int64_t kb_group, int64_t nb1_0) { + const int64_t sb = kb_group >> 3; // super-block index + const int g = (int) (kb_group & 7); // group within super-block + const int c = g >> 2; // 128-element chunk (0 or 1) + const int sub = g & 3; // sub-group within chunk (0..3) + + const int hi = sub >> 1; // high nibble for sub 2,3 + const int qloff = 64 * c + (sub & 1) * 32; + const int qhoff = 32 * c; + const int qhshift = 2 * sub; + const int scoff = 8 * c + 2 * sub; + + for (int j = 0; j < TILE_M; ++j) { + const block_q6_K *blk = + (const block_q6_K *)(src0_batch + (mb + j) * nb1_0) + sb; + const float d = ME_FP16(blk->d); + const uint8_t * ql = blk->ql + qloff; + const uint8_t * qh = blk->qh + qhoff; + const int8_t * sc = blk->scales + scoff; + + for (int l = 0; l < 32; ++l) { + const uint8_t nib = hi ? (ql[l] >> 4) : (ql[l] & 0xF); + const int8_t q = (int8_t)(nib | (((qh[l] >> qhshift) & 3) << 4)) - 32; + const float val = d * (float) sc[l >> 4] * (float) q; + panel[l * TILE_M + j] = val; + } + } +} + +// Spill / seed the FP32 C accumulator (16x16 tile in the vector register file, +// row n -> f2n[cols 0..7], f2n+1[cols 8..15]) to/from a 1 KB L2-SCP scratch. +// scratch layout: row n at byte offset n*64. Always moves all 16 rows; rows +// beyond a partial n_cur carry harmless garbage (never stored / recomputed). +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); +} + +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} + +int entry_point(struct ggml_et_binary_params *params, void *env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + // Dimensions (both harts need these for tile assignment) + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + if ((M % TILE_M) != 0) return 0; + if ((K % QK_K) != 0) return 0; + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t k_steps = K / BLOCK_K; // number of 32-element groups + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE; + const int64_t local_tile_idx = local_minion; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + // L2 SCP pointers for this minion. + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // Calculate ru_n to perfectly minimize hardware waves while avoiding Consumer bottleneck. + // The pipeline is perfectly balanced at r=8. Score = waves * max(8, r). + // We find the r that minimizes Score. + int64_t best_r = 1; + int64_t min_score = INT64_MAX; + int64_t max_search_r = REUSE_MAX; + if (max_search_r > n_tiles) max_search_r = n_tiles; + + for (int64_t r = 1; r <= max_search_r; r++) { + int64_t n_groups = (n_tiles + r - 1) / r; + int64_t base_units = m_tiles * n_groups * batch_count; + int64_t waves = (base_units + MACHINE_SLOTS - 1) / MACHINE_SLOTS; + + int64_t penalty = (r > 8) ? r : 8; + int64_t score = waves * penalty; + + if (score < min_score) { + min_score = score; + best_r = r; + } + } + int64_t ru_n = best_r; + + // Reuse pays only when it groups >=2 N-tiles; otherwise the windowing / + // C round-trip is pure overhead, so use the one-tile-at-a-time path. + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: dequant each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t n_groups = (n_tiles + ru_n - 1) / ru_n; + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + const int64_t n_windows = (k_steps + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + dequant_q6_K_panel(cf + i * (SCP_PANEL_SIZE / 4), + src0_batch, mb, kb0 + i, nb1_0); + } + FENCE; + flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } + + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; // first N-tile + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + const int is_last = (kw == n_windows - 1); + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + if (kw > 0) c_seed(cs); + int first = (kw == 0) ? 1 : 0; + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + for (int64_t i = 0; i < kbn; ++i) { + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = (kb0 + i) * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(cf + i * (SCP_PANEL_SIZE / 4) + half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + } + + if (is_last) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } + + // ===================================================================== + // ORIGINAL path: one output tile at a time (N % TILE_N != 0). No reuse. + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { + (float *) et_shire_l2scp_local(scp_base), + (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), + }; + + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); + + dequant_q6_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_panel[buf], BLOCK_K, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + FENCE; + return 0; + } + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t chunk_id = 0; + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + int first = 1; + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + chunk_id++; + scp_wait(ready_ctr, chunk_id); + + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = kb * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + + scp_signal(consumed_ctr, chunk_id); + } + + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c index 923acb7b07e0..49ed9700a4ee 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c @@ -37,25 +37,6 @@ typedef uint16_t et_fp16_t; #define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 2112 #define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 2176 -// Signal a counter value to the other hart via L2 SCP. -static inline void __attribute__((always_inline)) -scp_signal(volatile uint32_t *flag, uint32_t value) { - *flag = value; - FENCE; - evict_to_l2((const void *)flag, 1, 64); - WAIT_CACHEOPS; -} - -// Wait for a counter in L2 SCP to reach the expected value. -static inline void __attribute__((always_inline)) -scp_wait(volatile uint32_t *flag, uint32_t expected) { - while (1) { - evict_to_l2((const void *)flag, 1, 64); - WAIT_CACHEOPS; - if (*flag >= expected) return; - } -} - /** * Build the interleaved B panel that TensorFMA16A32 expects (vectorized). * diff --git a/ggml/src/ggml-et/et-kernels/src/platform.h b/ggml/src/ggml-et/et-kernels/src/platform.h index a36b99a36083..60efb0b6223f 100644 --- a/ggml/src/ggml-et/et-kernels/src/platform.h +++ b/ggml/src/ggml-et/et-kernels/src/platform.h @@ -632,4 +632,27 @@ static void evict_region_past_l2(const void * addr, size_t bytes) { } } +//****************************************************************************** +// Counter signaling between harts via L2 scratchpad (SCP) +//****************************************************************************** + +// Signal a counter value to the other hart via L2 SCP. +static inline void __attribute__((always_inline)) +scp_signal(volatile uint32_t *flag, uint32_t value) { + *flag = value; + FENCE; + evict_to_l2((const void *)flag, 1, 64); + WAIT_CACHEOPS; +} + +// Wait for a counter in L2 SCP to reach the expected value. +static inline void __attribute__((always_inline)) +scp_wait(volatile uint32_t *flag, uint32_t expected) { + while (1) { + evict_to_l2((const void *)flag, 1, 64); + WAIT_CACHEOPS; + if (*flag >= expected) return; + } +} + #endif // PLATFORM_H diff --git a/ggml/src/ggml-et/et-kernels/src/quants.h b/ggml/src/ggml-et/et-kernels/src/quants.h index 692ca00defe8..bb33a1e6c32e 100644 --- a/ggml/src/ggml-et/et-kernels/src/quants.h +++ b/ggml/src/ggml-et/et-kernels/src/quants.h @@ -33,6 +33,69 @@ static inline void dequantize_q4_0_block(const block_q4_0 * block, float * dst) } } +// Unpack the 12-byte packed Q3_K block scales into 16 signed 6-bit values. +static inline void unpack_q3_K_scales(const uint8_t * packed, int8_t * out /* [16] */) { + const uint32_t kmask1 = 0x03030303; + const uint32_t kmask2 = 0x0f0f0f0f; + uint32_t aux[4]; + __builtin_memcpy(aux, packed, 12); + const uint32_t tmp = aux[2]; + aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4); + aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4); + aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4); + aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4); + __builtin_memcpy(out, aux, 16); +} + +// Dequantize one Q2_K super-block (256 elements) to F32. +static inline void dequantize_q2_K_block(const block_q2_K * block, float * dst) { + const float d = fp16_to_fp32(block->d); + const float min = fp16_to_fp32(block->dmin); + const uint8_t * q = block->qs; + + int is = 0; + for (int n = 0; n < QK_K; n += 128) { + int shift = 0; + for (int j = 0; j < 4; ++j) { + uint8_t sc = block->scales[is++]; + float dl = d * (sc & 0xF), ml = min * (sc >> 4); + for (int l = 0; l < 16; ++l) *dst++ = dl * ((int8_t)((q[l] >> shift) & 3)) - ml; + sc = block->scales[is++]; + dl = d * (sc & 0xF); ml = min * (sc >> 4); + for (int l = 0; l < 16; ++l) *dst++ = dl * ((int8_t)((q[l + 16] >> shift) & 3)) - ml; + shift += 2; + } + q += 32; + } +} + +// Dequantize one Q3_K super-block (256 elements) to F32. +static inline void dequantize_q3_K_block(const block_q3_K * block, float * dst) { + const float d_all = fp16_to_fp32(block->d); + const uint8_t * q = block->qs; + const uint8_t * hm = block->hmask; + uint8_t m = 1; + + int8_t scales[16]; + unpack_q3_K_scales(block->scales, scales); + + int is = 0; + for (int n = 0; n < QK_K; n += 128) { + int shift = 0; + for (int j = 0; j < 4; ++j) { + float dl = d_all * (scales[is++] - 32); + for (int l = 0; l < 16; ++l) + *dst++ = dl * ((int8_t)((q[l + 0] >> shift) & 3) - ((hm[l + 0] & m) ? 0 : 4)); + dl = d_all * (scales[is++] - 32); + for (int l = 0; l < 16; ++l) + *dst++ = dl * ((int8_t)((q[l + 16] >> shift) & 3) - ((hm[l + 16] & m) ? 0 : 4)); + shift += 2; + m <<= 1; + } + q += 32; + } +} + // Unpack the 6-bit scale/min pair for Q4_K group j (groups 4-7 split their high bits). static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { if (j < 4) { @@ -69,4 +132,54 @@ static inline void dequantize_q4_K_block(const block_q4_K * block, float * dst) } } +// Dequantize one Q5_K super-block (256 elements) to F32. Same affine form as +// Q4_K with an extra high bit per weight drawn from qh. +static inline void dequantize_q5_K_block(const block_q5_K * block, float * dst) { + const uint8_t * ql = block->qs; + const uint8_t * qh = block->qh; + const float d = fp16_to_fp32(block->d); + const float min = fp16_to_fp32(block->dmin); + + int is = 0; + uint8_t sc, m; + uint8_t u1 = 1, u2 = 2; + for (int j = 0; j < QK_K; j += 64) { + get_scale_min_k4(is + 0, block->scales, &sc, &m); + const float d1 = d * sc, m1 = min * m; + get_scale_min_k4(is + 1, block->scales, &sc, &m); + const float d2 = d * sc, m2 = min * m; + for (int l = 0; l < 32; ++l) *dst++ = d1 * ((ql[l] & 0xF) + (qh[l] & u1 ? 16 : 0)) - m1; + for (int l = 0; l < 32; ++l) *dst++ = d2 * ((ql[l] >> 4) + (qh[l] & u2 ? 16 : 0)) - m2; + ql += 32; is += 2; + u1 <<= 2; u2 <<= 2; + } +} + +// Dequantize one Q6_K super-block (256 elements) to F32. Each 6-bit weight is +// (ql nibble | qh 2-bit) - 32, scaled by an int8 per-16 scale and the fp16 d. +static inline void dequantize_q6_K_block(const block_q6_K * block, float * dst) { + const float d = fp16_to_fp32(block->d); + const uint8_t * ql = block->ql; + const uint8_t * qh = block->qh; + const int8_t * sc = block->scales; + + for (int n = 0; n < QK_K; n += 128) { + for (int l = 0; l < 32; ++l) { + const int is = l / 16; + const int8_t q1 = (int8_t)((ql[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32; + const int8_t q2 = (int8_t)((ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32; + const int8_t q3 = (int8_t)((ql[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32; + const int8_t q4 = (int8_t)((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32; + dst[l + 0] = d * sc[is + 0] * q1; + dst[l + 32] = d * sc[is + 2] * q2; + dst[l + 64] = d * sc[is + 4] * q3; + dst[l + 96] = d * sc[is + 6] * q4; + } + dst += 128; + ql += 64; + qh += 32; + sc += 8; + } +} + #endif // QUANTS_H diff --git a/ggml/src/ggml-et/et-kernels/src/uberkernel.c b/ggml/src/ggml-et/et-kernels/src/uberkernel.c index 40d1cf9daa9b..af6c77cd8119 100644 --- a/ggml/src/ggml-et/et-kernels/src/uberkernel.c +++ b/ggml/src/ggml-et/et-kernels/src/uberkernel.c @@ -81,6 +81,11 @@ extern int mul_mat_f32_entry(struct ggml_et_binary_params *, void *); extern int mul_mat_f32_matrix_engine_entry(struct ggml_et_binary_params *, void *); extern int mul_mat_Q8_0_entry(struct ggml_et_mm_q8_params *, void *); extern int mul_mat_Q4_0_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_Q4_K_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_Q2_K_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_Q3_K_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_Q5_K_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_Q6_K_entry(struct ggml_et_binary_params *, void *); static inline size_t tensor_bytes(const struct ggml_tensor * t) { return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; @@ -483,6 +488,36 @@ int entry_point(struct ggml_et_uberkernel_params * params, void * env) { rc = mul_mat_Q4_0_entry(p, env); break; } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_Q4_K: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_Q4_K_entry(p, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_Q2_K: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_Q2_K_entry(p, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_Q3_K: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_Q3_K_entry(p, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_Q5_K: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_Q5_K_entry(p, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_Q6_K: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_Q6_K_entry(p, env); + break; + } default: return -1; diff --git a/ggml/src/ggml-et/ggml-et-ops.cpp b/ggml/src/ggml-et/ggml-et-ops.cpp index 4a15fad2b538..013cb59bc2d8 100644 --- a/ggml/src/ggml-et/ggml-et-ops.cpp +++ b/ggml/src/ggml-et/ggml-et-ops.cpp @@ -743,6 +743,46 @@ bool ggml_et_op_mul_mat(ggml_backend_et_device_context * dev_ctx, kernel_name = "mul_mat_Q4_K"; // N < 53, or M % 16 != 0, or K % 256 != 0 src0_type_name = "Q4_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q6_K && + node->src[1]->type == GGML_TYPE_F32 && + node->src[1]->ne[1] >= 53 && // N >= 53 (prefill): use matrix engine + node->src[0]->ne[1] % 16 == 0 && // M % TILE_M + node->src[0]->ne[0] % 256 == 0) { // K % QK_K (Q6_K super-block) + kernel_name = "mul_mat_Q6_K_matrix_engine"; + src0_type_name = "Q6_K"; + } else if (node->type == GGML_TYPE_F32 && + node->src[0]->type == GGML_TYPE_Q6_K && + node->src[1]->type == GGML_TYPE_F32) { + + kernel_name = "mul_mat_Q6_K"; // N < 53, or M % 16 != 0, or K % 256 != 0 + src0_type_name = "Q6_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q2_K && + node->src[1]->type == GGML_TYPE_F32 && + node->src[1]->ne[1] >= 53 && node->src[0]->ne[1] % 16 == 0 && node->src[0]->ne[0] % 256 == 0) { + kernel_name = "mul_mat_Q2_K_matrix_engine"; + src0_type_name = "Q2_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q2_K && + node->src[1]->type == GGML_TYPE_F32) { + kernel_name = "mul_mat_Q2_K"; // N < 53, or M % 16 != 0, or K % 256 != 0 + src0_type_name = "Q2_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q3_K && + node->src[1]->type == GGML_TYPE_F32 && + node->src[1]->ne[1] >= 53 && node->src[0]->ne[1] % 16 == 0 && node->src[0]->ne[0] % 256 == 0) { + kernel_name = "mul_mat_Q3_K_matrix_engine"; + src0_type_name = "Q3_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q3_K && + node->src[1]->type == GGML_TYPE_F32) { + kernel_name = "mul_mat_Q3_K"; // N < 53, or M % 16 != 0, or K % 256 != 0 + src0_type_name = "Q3_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q5_K && + node->src[1]->type == GGML_TYPE_F32 && + node->src[1]->ne[1] >= 53 && node->src[0]->ne[1] % 16 == 0 && node->src[0]->ne[0] % 256 == 0) { + kernel_name = "mul_mat_Q5_K_matrix_engine"; + src0_type_name = "Q5_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q5_K && + node->src[1]->type == GGML_TYPE_F32) { + kernel_name = "mul_mat_Q5_K"; // N < 53, or M % 16 != 0, or K % 256 != 0 + src0_type_name = "Q5_K"; } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q8_0 && node->src[1]->type == GGML_TYPE_F32) { kernel_name = "mul_mat_Q8_0"; @@ -1633,7 +1673,9 @@ bool ggml_et_op_get_rows(ggml_backend_et_device_context * dev_ctx, const ggml_te if (node->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_I32 && (node->src[0]->type == GGML_TYPE_F32 || node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_Q4_0 || node->src[0]->type == GGML_TYPE_Q8_0 || - node->src[0]->type == GGML_TYPE_Q4_K)) { + node->src[0]->type == GGML_TYPE_Q4_K || node->src[0]->type == GGML_TYPE_Q6_K || + node->src[0]->type == GGML_TYPE_Q2_K || node->src[0]->type == GGML_TYPE_Q3_K || + node->src[0]->type == GGML_TYPE_Q5_K)) { kernel_name = "get_rows_f32"; } else { diff --git a/ggml/src/ggml-et/ggml-et.cpp b/ggml/src/ggml-et/ggml-et.cpp index 013d3db5adb8..6efb10a6a83f 100644 --- a/ggml/src/ggml-et/ggml-et.cpp +++ b/ggml/src/ggml-et/ggml-et.cpp @@ -1033,6 +1033,26 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm } } + supported = src0_first_dim_contiguous && + src1_first_dim_contiguous && + dst_first_dim_contiguous && + dst_properly_ordered; + } else if (op->type == GGML_TYPE_F32 && + op->src[0] && (op->src[0]->type == GGML_TYPE_Q6_K || op->src[0]->type == GGML_TYPE_Q2_K || + op->src[0]->type == GGML_TYPE_Q3_K || op->src[0]->type == GGML_TYPE_Q5_K) && + op->src[1] && op->src[1]->type == GGML_TYPE_F32) { + + bool src0_first_dim_contiguous = (op->src[0]->nb[0] == ggml_type_size(op->src[0]->type)); + bool src1_first_dim_contiguous = (op->src[1]->nb[0] == ggml_type_size(op->src[1]->type)); + bool dst_first_dim_contiguous = (op->nb[0] == sizeof(float)); + + bool dst_properly_ordered = true; + for (int d = 0; d < 3; d++) { + if (op->ne[d] > 1 && op->ne[d+1] > 1 && op->nb[d] > op->nb[d+1]) { + dst_properly_ordered = false; + } + } + supported = src0_first_dim_contiguous && src1_first_dim_contiguous && dst_first_dim_contiguous && @@ -1339,11 +1359,13 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm } break; case GGML_OP_GET_ROWS: - // Support F32/F16/Q4_0/Q8_0/Q4_K data with I32 indices -> F32 output + // Support F32/F16/Q4_0/Q8_0/Q4_K/Q6_K data with I32 indices -> F32 output if (op->type == GGML_TYPE_F32 && op->src[0] && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 || - op->src[0]->type == GGML_TYPE_Q4_K) && + op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q6_K || + op->src[0]->type == GGML_TYPE_Q2_K || op->src[0]->type == GGML_TYPE_Q3_K || + op->src[0]->type == GGML_TYPE_Q5_K) && op->src[1] && op->src[1]->type == GGML_TYPE_I32 && ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) { // Validate dimension constraints from ggml implementation From ef1134dadf661650e7f94e637b29a8eca3f8a4e2 Mon Sep 17 00:00:00 2001 From: CodeDoes Date: Tue, 21 Jul 2026 20:14:38 +0200 Subject: [PATCH 12/12] fix: stable fgb.ps dot product and N-split for 2048 harts - fix: replace fg32b.ps with fgb.ps in q8_dot_tile and q8_dot_compute_x2_aligned fg32b.ps requires 32-byte aligned base, but block_q8_0.qs starts at offset 2 after fp16 d, causing intermittent Code 0 Type 4 stream error. - perf: add N-split path for generation (M<=4) to utilize all 2048 harts --- ggml/src/ggml-et/et-kernels/src/block_ops.h | 401 ++++-------------- .../src/ggml-et/et-kernels/src/mul_mat_Q8_0.c | 150 ++++--- 2 files changed, 190 insertions(+), 361 deletions(-) diff --git a/ggml/src/ggml-et/et-kernels/src/block_ops.h b/ggml/src/ggml-et/et-kernels/src/block_ops.h index 90023172ec46..9478882720e0 100644 --- a/ggml/src/ggml-et/et-kernels/src/block_ops.h +++ b/ggml/src/ggml-et/et-kernels/src/block_ops.h @@ -162,118 +162,36 @@ static inline void __attribute__((always_inline)) q8_dot_tile(const block_q8_0 * const float * b_col, int64_t n_blocks) { const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; - const uint64_t gather_0_to_7 = 0x398a418820ULL; __asm__ volatile("flw.ps f31, %[g]\n" : : [g] "m"(*(const int32_t (*)[8]) gather_pattern) : "f31"); for (int64_t kb = 0; kb < n_blocks; kb++) { - const block_q8_0 * blk = q_row + kb; - const float * b_ptr = b_col + (kb << 5); - const uintptr_t qs_addr = (uintptr_t) blk->qs; - const uintptr_t qs_aligned = qs_addr & ~(uintptr_t) 31; - const uintptr_t qs_low = qs_addr & 31; - const int fast_chunks = (int) ((32 - qs_low) >> 3); - - if (fast_chunks >= 3) { - __asm__ volatile( - "fbci.pi f10, 0\n" - "flw.ps f12, %[bv0]\n" - "fg32b.ps f11, %[gi](%[ap0])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv1]\n" - "fg32b.ps f11, %[gi](%[ap1])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv2]\n" - "fg32b.ps f11, %[gi](%[ap2])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv3]\n" - "fgb.ps f11, f31(%[ap3])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - : - : [gi] "r"(gather_0_to_7), [ap0] "r"(qs_addr), [ap1] "r"(qs_aligned | ((qs_addr + 8) & 31)), - [ap2] "r"(qs_aligned | ((qs_addr + 16) & 31)), [ap3] "r"(&blk->qs[24]), - [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), - [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12"); - } else if (fast_chunks == 2) { - __asm__ volatile( - "fbci.pi f10, 0\n" - "flw.ps f12, %[bv0]\n" - "fg32b.ps f11, %[gi](%[ap0])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv1]\n" - "fg32b.ps f11, %[gi](%[ap1])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv2]\n" - "fgb.ps f11, f31(%[ap2])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv3]\n" - "fgb.ps f11, f31(%[ap3])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - : - : [gi] "r"(gather_0_to_7), [ap0] "r"(qs_addr), [ap1] "r"(qs_aligned | ((qs_addr + 8) & 31)), - [ap2] "r"(&blk->qs[16]), [ap3] "r"(&blk->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), - [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), - [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12"); - } else if (fast_chunks == 1) { - __asm__ volatile( - "fbci.pi f10, 0\n" - "flw.ps f12, %[bv0]\n" - "fg32b.ps f11, %[gi](%[ap0])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv1]\n" - "fgb.ps f11, f31(%[ap1])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv2]\n" - "fgb.ps f11, f31(%[ap2])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv3]\n" - "fgb.ps f11, f31(%[ap3])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - : - : [gi] "r"(gather_0_to_7), [ap0] "r"(qs_addr), [ap1] "r"(&blk->qs[8]), [ap2] "r"(&blk->qs[16]), - [ap3] "r"(&blk->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), - [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), - [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12"); - } else { - __asm__ volatile( - "fbci.pi f10, 0\n" - "flw.ps f12, %[bv0]\n" - "fgb.ps f11, f31(%[ap0])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv1]\n" - "fgb.ps f11, f31(%[ap1])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv2]\n" - "fgb.ps f11, f31(%[ap2])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv3]\n" - "fgb.ps f11, f31(%[ap3])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - : - : [ap0] "r"(&blk->qs[0]), [ap1] "r"(&blk->qs[8]), [ap2] "r"(&blk->qs[16]), [ap3] "r"(&blk->qs[24]), - [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), - [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12"); - } + const block_q8_0 * blk = q_row + kb; + const float * b_ptr = b_col + (kb << 5); + + __asm__ volatile( + "fbci.pi f10, 0\n" + "flw.ps f12, %[bv0]\n" + "fgb.ps f11, f31(%[ap0])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv1]\n" + "fgb.ps f11, f31(%[ap1])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv2]\n" + "fgb.ps f11, f31(%[ap2])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv3]\n" + "fgb.ps f11, f31(%[ap3])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + : + : [ap0] "r"(&blk->qs[0]), [ap1] "r"(&blk->qs[8]), [ap2] "r"(&blk->qs[16]), [ap3] "r"(&blk->qs[24]), + [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), + [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12"); // f20 += f10 * broadcast(scale) — hardware fp16→fp32 via FCVT.PS.F16 uint32_t scale_raw = (uint32_t) blk->d; @@ -358,8 +276,7 @@ static inline void q8_dot_compute_x2_aligned(const block_q8_0 * q_row0, int64_t K_blocks, float * out0, float * out1) { - const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; - const uint64_t gather_0_to_7 = 0x398a418820ULL; + const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; __asm__ volatile("flw.ps f31, %[g]\n" : : [g] "m"(*(const int32_t (*)[8]) gather_pattern) : "f31"); __asm__ volatile( "fbci.pi f20, 0\n" @@ -371,225 +288,83 @@ static inline void q8_dot_compute_x2_aligned(const block_q8_0 * q_row0, const block_q8_0 * blk1 = q_row1 + kb; const float * b_ptr = b_col + (kb << 5); - const uintptr_t qs_addr0 = (uintptr_t) blk0->qs; - const uintptr_t qs_addr1 = (uintptr_t) blk1->qs; - const uintptr_t qs_aligned0 = qs_addr0 & ~(uintptr_t) 31; - const uintptr_t qs_aligned1 = qs_addr1 & ~(uintptr_t) 31; - const int fast_chunks = (int) ((32 - (qs_addr0 & 31)) >> 3); - - if (fast_chunks >= 3) { - __asm__ volatile( - "fbci.pi f10, 0\n" - "fbci.pi f11, 0\n" - - "flw.ps f12, %[bv0]\n" - "fg32b.ps f16, %[gi](%[r0ap0])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f12, f10\n" - "fg32b.ps f17, %[gi](%[r1ap0])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f12, f11\n" - - "flw.ps f13, %[bv1]\n" - "fg32b.ps f16, %[gi](%[r0ap1])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f13, f10\n" - "fg32b.ps f17, %[gi](%[r1ap1])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f13, f11\n" - - "flw.ps f14, %[bv2]\n" - "fg32b.ps f16, %[gi](%[r0ap2])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f14, f10\n" - "fg32b.ps f17, %[gi](%[r1ap2])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f14, f11\n" - - "flw.ps f15, %[bv3]\n" - "fgb.ps f16, f31(%[r0ap3])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f15, f10\n" - "fgb.ps f17, f31(%[r1ap3])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f15, f11\n" - : - : [gi] "r"(gather_0_to_7), [r0ap0] "r"(qs_addr0), [r0ap1] "r"(qs_aligned0 | ((qs_addr0 + 8) & 31)), - [r0ap2] "r"(qs_aligned0 | ((qs_addr0 + 16) & 31)), [r0ap3] "r"(&blk0->qs[24]), [r1ap0] "r"(qs_addr1), - [r1ap1] "r"(qs_aligned1 | ((qs_addr1 + 8) & 31)), [r1ap2] "r"(qs_aligned1 | ((qs_addr1 + 16) & 31)), - [r1ap3] "r"(&blk1->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), - [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), - [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); - } else if (fast_chunks == 2) { - __asm__ volatile( - "fbci.pi f10, 0\n" - "fbci.pi f11, 0\n" - - "flw.ps f12, %[bv0]\n" - "fg32b.ps f16, %[gi](%[r0ap0])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f12, f10\n" - "fg32b.ps f17, %[gi](%[r1ap0])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f12, f11\n" - - "flw.ps f13, %[bv1]\n" - "fg32b.ps f16, %[gi](%[r0ap1])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f13, f10\n" - "fg32b.ps f17, %[gi](%[r1ap1])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f13, f11\n" - - "flw.ps f14, %[bv2]\n" - "fgb.ps f16, f31(%[r0ap2])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f14, f10\n" - "fgb.ps f17, f31(%[r1ap2])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f14, f11\n" - - "flw.ps f15, %[bv3]\n" - "fgb.ps f16, f31(%[r0ap3])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f15, f10\n" - "fgb.ps f17, f31(%[r1ap3])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f15, f11\n" - : - : [gi] "r"(gather_0_to_7), [r0ap0] "r"(qs_addr0), [r0ap1] "r"(qs_aligned0 | ((qs_addr0 + 8) & 31)), - [r0ap2] "r"(&blk0->qs[16]), [r0ap3] "r"(&blk0->qs[24]), [r1ap0] "r"(qs_addr1), - [r1ap1] "r"(qs_aligned1 | ((qs_addr1 + 8) & 31)), [r1ap2] "r"(&blk1->qs[16]), - [r1ap3] "r"(&blk1->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), - [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), - [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); - } else if (fast_chunks == 1) { - __asm__ volatile( - "fbci.pi f10, 0\n" - "fbci.pi f11, 0\n" - - "flw.ps f12, %[bv0]\n" - "fg32b.ps f16, %[gi](%[r0ap0])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f12, f10\n" - "fg32b.ps f17, %[gi](%[r1ap0])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f12, f11\n" - - "flw.ps f13, %[bv1]\n" - "fgb.ps f16, f31(%[r0ap1])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f13, f10\n" - "fgb.ps f17, f31(%[r1ap1])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f13, f11\n" - - "flw.ps f14, %[bv2]\n" - "fgb.ps f16, f31(%[r0ap2])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f14, f10\n" - "fgb.ps f17, f31(%[r1ap2])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f14, f11\n" - - "flw.ps f15, %[bv3]\n" - "fgb.ps f16, f31(%[r0ap3])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f15, f10\n" - "fgb.ps f17, f31(%[r1ap3])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f15, f11\n" - : - : [gi] "r"(gather_0_to_7), [r0ap0] "r"(qs_addr0), [r0ap1] "r"(&blk0->qs[8]), [r0ap2] "r"(&blk0->qs[16]), - [r0ap3] "r"(&blk0->qs[24]), [r1ap0] "r"(qs_addr1), [r1ap1] "r"(&blk1->qs[8]), - [r1ap2] "r"(&blk1->qs[16]), [r1ap3] "r"(&blk1->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), - [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), - [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); - } else { - __asm__ volatile( - "fbci.pi f10, 0\n" - "fbci.pi f11, 0\n" - - "flw.ps f12, %[bv0]\n" - "fgb.ps f16, f31(%[r0ap0])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f12, f10\n" - "fgb.ps f17, f31(%[r1ap0])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f12, f11\n" - - "flw.ps f13, %[bv1]\n" - "fgb.ps f16, f31(%[r0ap1])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f13, f10\n" - "fgb.ps f17, f31(%[r1ap1])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f13, f11\n" - - "flw.ps f14, %[bv2]\n" - "fgb.ps f16, f31(%[r0ap2])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f14, f10\n" - "fgb.ps f17, f31(%[r1ap2])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f14, f11\n" - - "flw.ps f15, %[bv3]\n" - "fgb.ps f16, f31(%[r0ap3])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f15, f10\n" - "fgb.ps f17, f31(%[r1ap3])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f15, f11\n" - : - : [r0ap0] "r"(&blk0->qs[0]), [r0ap1] "r"(&blk0->qs[8]), [r0ap2] "r"(&blk0->qs[16]), - [r0ap3] "r"(&blk0->qs[24]), [r1ap0] "r"(&blk1->qs[0]), [r1ap1] "r"(&blk1->qs[8]), - [r1ap2] "r"(&blk1->qs[16]), [r1ap3] "r"(&blk1->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), - [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), - [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); - } - - const uint32_t scale_raw0 = (uint32_t) blk0->d; - const uint32_t scale_raw1 = (uint32_t) blk1->d; __asm__ volatile( - "fbcx.ps f24, %[s0]\n" - "fcvt.ps.f16 f24, f24\n" - "fmadd.ps f20, f10, f24, f20\n" - "fbcx.ps f25, %[s1]\n" - "fcvt.ps.f16 f25, f25\n" - "fmadd.ps f21, f11, f25, f21\n" + "fbci.pi f10, 0\n" + "fbci.pi f11, 0\n" + + "flw.ps f12, %[bv0]\n" + "fgb.ps f16, f31(%[r0ap0])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f12, f10\n" + "fgb.ps f17, f31(%[r1ap0])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f12, f11\n" + + "flw.ps f13, %[bv1]\n" + "fgb.ps f16, f31(%[r0ap1])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f13, f10\n" + "fgb.ps f17, f31(%[r1ap1])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f13, f11\n" + + "flw.ps f14, %[bv2]\n" + "fgb.ps f16, f31(%[r0ap2])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f14, f10\n" + "fgb.ps f17, f31(%[r1ap2])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f14, f11\n" + + "flw.ps f15, %[bv3]\n" + "fgb.ps f16, f31(%[r0ap3])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f15, f10\n" + "fgb.ps f17, f31(%[r1ap3])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f15, f11\n" + : + : [r0ap0] "r"(&blk0->qs[0]), [r0ap1] "r"(&blk0->qs[8]), [r0ap2] "r"(&blk0->qs[16]), [r0ap3] "r"(&blk0->qs[24]), + [r1ap0] "r"(&blk1->qs[0]), [r1ap1] "r"(&blk1->qs[8]), [r1ap2] "r"(&blk1->qs[16]), [r1ap3] "r"(&blk1->qs[24]), + [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), + [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); + + // f20 += f10 * broadcast(scale0); f21 += f11 * broadcast(scale1) + uint32_t scale_raw0 = (uint32_t) blk0->d; + uint32_t scale_raw1 = (uint32_t) blk1->d; + __asm__ volatile( + "fbcx.ps f18, %[s0]\n" + "fbcx.ps f19, %[s1]\n" + "fcvt.ps.f16 f18, f18\n" + "fcvt.ps.f16 f19, f19\n" + "fmadd.ps f20, f10, f18, f20\n" + "fmadd.ps f21, f11, f19, f21\n" : : [s0] "r"(scale_raw0), [s1] "r"(scale_raw1) - : "f20", "f21", "f24", "f25"); + : "f18", "f19", "f20", "f21"); } - - float result0; - float result1; - __asm__ __volatile__( + float s0, s1; + __asm__ volatile( "fswizz.ps f1, f20, 0xB1 \n\t" "fadd.ps f2, f20, f1, rne \n\t" "fswizz.ps f3, f2, 0x4E \n\t" "fadd.ps f4, f2, f3, rne \n\t" "fmvz.x.ps t0, f4, 4 \n\t" "fbcx.ps f5, t0 \n\t" - "fadd.ps %[vout], f4, f5, rne \n\t" - : [vout] "=f"(result0)::"t0", "f1", "f2", "f3", "f4", "f5"); - __asm__ __volatile__( + "fadd.ps %[v0], f4, f5, rne \n\t" "fswizz.ps f1, f21, 0xB1 \n\t" "fadd.ps f2, f21, f1, rne \n\t" "fswizz.ps f3, f2, 0x4E \n\t" "fadd.ps f4, f2, f3, rne \n\t" "fmvz.x.ps t0, f4, 4 \n\t" "fbcx.ps f5, t0 \n\t" - "fadd.ps %[vout], f4, f5, rne \n\t" - : [vout] "=f"(result1)::"t0", "f1", "f2", "f3", "f4", "f5"); - - *out0 = result0; - *out1 = result1; + "fadd.ps %[v1], f4, f5, rne \n\t" + : [v0] "=f"(s0), [v1] "=f"(s1) + : + : "t0", "f1", "f2", "f3", "f4", "f5", "f20", "f21"); + *out0 = s0; + *out1 = s1; } // Compute dot product between f16 block and f32 column vector (NAIVE VERSION) diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c index ad21a3ee043a..930d764ca6e3 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c @@ -335,59 +335,47 @@ int entry_point(struct ggml_et_mm_q8_params * params, void * env) { * When `nb01` is 32-byte aligned, every row has the same block-alignment * pattern. That lets us compute two rows together and reuse each loaded * B chunk across both rows instead of reloading it in a second dot call. + * + * For generation (M=1..4), M-split leaves most harts idle (only 1-4 + * active out of 2048). N-split distributes the N dimension across harts + * instead, keeping all N harts busy — a 100-2000x utilization gain. */ - for (int64_t i3 = 0; i3 < ne13; i3++) { - const int64_t i03 = i3 / r3; - const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; - const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; - char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; - const char * bias_ptr3 = bias_base ? bias_base + i3 * nbb3 : (const char *) 0; - - for (int64_t i2 = 0; i2 < ne12; i2++) { - const int64_t i02 = i2 / r2; - const char * src0_ptr2 = src0_ptr3 + i02 * nb02; - const char * src1_ptr2 = src1_ptr3 + i2 * nb12; - char * dst_ptr2 = dst_ptr3 + i2 * nbd2; - const char * bias_ptr2 = bias_ptr3 ? bias_ptr3 + i2 * nbb2 : (const char *) 0; - - for (int64_t n = 0; n < N; n++) { - const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); - const float * bias_n = bias_ptr2 ? (const float *) (bias_ptr2 + n * nbb1) : (const float *) 0; - q8_dot_state q8_state; + const int use_n_split = (M <= 4 && N > M && N >= 32); + if (use_n_split) { + /* + * N-split: distribute output columns across harts. + * Each hart processes ALL rows (M) for its subset of columns. + * This keeps many harts active during generation where M=1. + */ + const int64_t total_harts = 2048; + const int64_t n_per_hart = (N + total_harts - 1) / total_harts; + const int64_t n_start = hart_id * n_per_hart; + const int64_t n_end = (n_start + n_per_hart < N) ? (n_start + n_per_hart) : N; + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; + const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; + char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; + const char * bias_ptr3 = bias_base ? bias_base + i3 * nbb3 : (const char *) 0; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char * src0_ptr2 = src0_ptr3 + i02 * nb02; + const char * src1_ptr2 = src1_ptr3 + i2 * nb12; + char * dst_ptr2 = dst_ptr3 + i2 * nbd2; + const char * bias_ptr2 = bias_ptr3 ? bias_ptr3 + i2 * nbb2 : (const char *) 0; + + q8_dot_state q8_state; q8_dot_begin(&q8_state); - if (use_simple_x2) { - for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * SIMPLE_X2_ROWS) { - const int64_t m1 = m0 + STRIDE_M; - const block_q8_0 * q_row0 = (const block_q8_0 *) (src0_ptr2 + m0 * nb01); - - if (m1 < M) { - const block_q8_0 * q_row1 = (const block_q8_0 *) (src0_ptr2 + m1 * nb01); - float s0, s1; - q8_dot_compute_x2_aligned(q_row0, q_row1, b_col_base, K_blocks, &s0, &s1); + for (int64_t n = n_start; n < n_end; n++) { + const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); + const float * bias_n = bias_ptr2 ? (const float *) (bias_ptr2 + n * nbb1) : (const float *) 0; - float * dst0 = (float *) (dst_ptr2 + n * nbd1 + m0 * sizeof(float)); - float * dst1 = (float *) (dst_ptr2 + n * nbd1 + m1 * sizeof(float)); - if (bias_n) { - s0 += bias_n[m0]; - s1 += bias_n[m1]; - } - atomic_store_f32((volatile float *) dst0, s0); - atomic_store_f32((volatile float *) dst1, s1); - } else { - float sum = q8_dot_compute(q_row0, b_col_base, K_blocks); - float * dst = (float *) (dst_ptr2 + n * nbd1 + m0 * sizeof(float)); - if (bias_n) { - sum += bias_n[m0]; - } - atomic_store_f32((volatile float *) dst, sum); - } - } - } else { - for (int64_t m = hart_id; m < M; m += STRIDE_M) { + for (int64_t m = 0; m < M; m++) { const block_q8_0 * q_row = (const block_q8_0 *) (src0_ptr2 + m * nb01); - - float sum = q8_dot_compute(q_row, b_col_base, K_blocks); + float sum = q8_dot_compute(q_row, b_col_base, K_blocks); float * dst_entry = (float *) (dst_ptr2 + n * nbd1 + m * sizeof(float)); if (bias_n) { @@ -400,6 +388,72 @@ int entry_point(struct ggml_et_mm_q8_params * params, void * env) { q8_dot_end(&q8_state); } } + } else { + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; + const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; + char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; + const char * bias_ptr3 = bias_base ? bias_base + i3 * nbb3 : (const char *) 0; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char * src0_ptr2 = src0_ptr3 + i02 * nb02; + const char * src1_ptr2 = src1_ptr3 + i2 * nb12; + char * dst_ptr2 = dst_ptr3 + i2 * nbd2; + const char * bias_ptr2 = bias_ptr3 ? bias_ptr3 + i2 * nbb2 : (const char *) 0; + + for (int64_t n = 0; n < N; n++) { + const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); + const float * bias_n = bias_ptr2 ? (const float *) (bias_ptr2 + n * nbb1) : (const float *) 0; + q8_dot_state q8_state; + q8_dot_begin(&q8_state); + + if (use_simple_x2) { + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * SIMPLE_X2_ROWS) { + const int64_t m1 = m0 + STRIDE_M; + const block_q8_0 * q_row0 = (const block_q8_0 *) (src0_ptr2 + m0 * nb01); + + if (m1 < M) { + const block_q8_0 * q_row1 = (const block_q8_0 *) (src0_ptr2 + m1 * nb01); + float s0, s1; + q8_dot_compute_x2_aligned(q_row0, q_row1, b_col_base, K_blocks, &s0, &s1); + + float * dst0 = (float *) (dst_ptr2 + n * nbd1 + m0 * sizeof(float)); + float * dst1 = (float *) (dst_ptr2 + n * nbd1 + m1 * sizeof(float)); + if (bias_n) { + s0 += bias_n[m0]; + s1 += bias_n[m1]; + } + atomic_store_f32((volatile float *) dst0, s0); + atomic_store_f32((volatile float *) dst1, s1); + } else { + float sum = q8_dot_compute(q_row0, b_col_base, K_blocks); + float * dst = (float *) (dst_ptr2 + n * nbd1 + m0 * sizeof(float)); + if (bias_n) { + sum += bias_n[m0]; + } + atomic_store_f32((volatile float *) dst, sum); + } + } + } else { + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q8_0 * q_row = (const block_q8_0 *) (src0_ptr2 + m * nb01); + + float sum = q8_dot_compute(q_row, b_col_base, K_blocks); + + float * dst_entry = (float *) (dst_ptr2 + n * nbd1 + m * sizeof(float)); + if (bias_n) { + sum += bias_n[m]; + } + atomic_store_f32((volatile float *) dst_entry, sum); + } + } + + q8_dot_end(&q8_state); + } + } + } } }