diff --git a/ggml/src/ggml-et/CMakeLists.txt b/ggml/src/ggml-et/CMakeLists.txt index ee0ee3759a91..a11d0cd3b5e7 100644 --- a/ggml/src/ggml-et/CMakeLists.txt +++ b/ggml/src/ggml-et/CMakeLists.txt @@ -38,6 +38,16 @@ set(KERNELS mul_mat_Q8_0 mul_mat_Q4_0 mul_mat_Q4_0_matrix_engine + mul_mat_Q4_K + mul_mat_Q4_K_matrix_engine + mul_mat_Q6_K + mul_mat_Q6_K_matrix_engine + mul_mat_Q2_K + mul_mat_Q2_K_matrix_engine + mul_mat_Q3_K + mul_mat_Q3_K_matrix_engine + mul_mat_Q5_K + mul_mat_Q5_K_matrix_engine mul_mat_f16 mul_mat_f16_matrix_engine rope_f32 @@ -103,6 +113,11 @@ set(UBERKERNEL_SUPPORTED_KERNELS mul_mat_f32_matrix_engine mul_mat_id_f32 mul_mat_Q4_0 + mul_mat_Q4_K + mul_mat_Q2_K + mul_mat_Q3_K + mul_mat_Q5_K + mul_mat_Q6_K mul_mat_Q8_0 norm_f32 pad_f32 diff --git a/ggml/src/ggml-et/et-kernels/src/block_ops.h b/ggml/src/ggml-et/et-kernels/src/block_ops.h index 78ffbde87bfa..9478882720e0 100644 --- a/ggml/src/ggml-et/et-kernels/src/block_ops.h +++ b/ggml/src/ggml-et/et-kernels/src/block_ops.h @@ -162,118 +162,36 @@ static inline void __attribute__((always_inline)) q8_dot_tile(const block_q8_0 * const float * b_col, int64_t n_blocks) { const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; - const uint64_t gather_0_to_7 = 0x398a418820ULL; __asm__ volatile("flw.ps f31, %[g]\n" : : [g] "m"(*(const int32_t (*)[8]) gather_pattern) : "f31"); for (int64_t kb = 0; kb < n_blocks; kb++) { - const block_q8_0 * blk = q_row + kb; - const float * b_ptr = b_col + (kb << 5); - const uintptr_t qs_addr = (uintptr_t) blk->qs; - const uintptr_t qs_aligned = qs_addr & ~(uintptr_t) 31; - const uintptr_t qs_low = qs_addr & 31; - const int fast_chunks = (int) ((32 - qs_low) >> 3); - - if (fast_chunks >= 3) { - __asm__ volatile( - "fbci.pi f10, 0\n" - "flw.ps f12, %[bv0]\n" - "fg32b.ps f11, %[gi](%[ap0])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv1]\n" - "fg32b.ps f11, %[gi](%[ap1])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv2]\n" - "fg32b.ps f11, %[gi](%[ap2])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv3]\n" - "fgb.ps f11, f31(%[ap3])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - : - : [gi] "r"(gather_0_to_7), [ap0] "r"(qs_addr), [ap1] "r"(qs_aligned | ((qs_addr + 8) & 31)), - [ap2] "r"(qs_aligned | ((qs_addr + 16) & 31)), [ap3] "r"(&blk->qs[24]), - [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), - [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12"); - } else if (fast_chunks == 2) { - __asm__ volatile( - "fbci.pi f10, 0\n" - "flw.ps f12, %[bv0]\n" - "fg32b.ps f11, %[gi](%[ap0])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv1]\n" - "fg32b.ps f11, %[gi](%[ap1])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv2]\n" - "fgb.ps f11, f31(%[ap2])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv3]\n" - "fgb.ps f11, f31(%[ap3])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - : - : [gi] "r"(gather_0_to_7), [ap0] "r"(qs_addr), [ap1] "r"(qs_aligned | ((qs_addr + 8) & 31)), - [ap2] "r"(&blk->qs[16]), [ap3] "r"(&blk->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), - [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), - [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12"); - } else if (fast_chunks == 1) { - __asm__ volatile( - "fbci.pi f10, 0\n" - "flw.ps f12, %[bv0]\n" - "fg32b.ps f11, %[gi](%[ap0])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv1]\n" - "fgb.ps f11, f31(%[ap1])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv2]\n" - "fgb.ps f11, f31(%[ap2])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv3]\n" - "fgb.ps f11, f31(%[ap3])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - : - : [gi] "r"(gather_0_to_7), [ap0] "r"(qs_addr), [ap1] "r"(&blk->qs[8]), [ap2] "r"(&blk->qs[16]), - [ap3] "r"(&blk->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), - [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), - [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12"); - } else { - __asm__ volatile( - "fbci.pi f10, 0\n" - "flw.ps f12, %[bv0]\n" - "fgb.ps f11, f31(%[ap0])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv1]\n" - "fgb.ps f11, f31(%[ap1])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv2]\n" - "fgb.ps f11, f31(%[ap2])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - "flw.ps f12, %[bv3]\n" - "fgb.ps f11, f31(%[ap3])\n" - "fcvt.ps.pw f11, f11\n" - "fmadd.ps f10, f11, f12, f10\n" - : - : [ap0] "r"(&blk->qs[0]), [ap1] "r"(&blk->qs[8]), [ap2] "r"(&blk->qs[16]), [ap3] "r"(&blk->qs[24]), - [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), - [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12"); - } + const block_q8_0 * blk = q_row + kb; + const float * b_ptr = b_col + (kb << 5); + + __asm__ volatile( + "fbci.pi f10, 0\n" + "flw.ps f12, %[bv0]\n" + "fgb.ps f11, f31(%[ap0])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv1]\n" + "fgb.ps f11, f31(%[ap1])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv2]\n" + "fgb.ps f11, f31(%[ap2])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv3]\n" + "fgb.ps f11, f31(%[ap3])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + : + : [ap0] "r"(&blk->qs[0]), [ap1] "r"(&blk->qs[8]), [ap2] "r"(&blk->qs[16]), [ap3] "r"(&blk->qs[24]), + [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), + [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12"); // f20 += f10 * broadcast(scale) — hardware fp16→fp32 via FCVT.PS.F16 uint32_t scale_raw = (uint32_t) blk->d; @@ -358,8 +276,7 @@ static inline void q8_dot_compute_x2_aligned(const block_q8_0 * q_row0, int64_t K_blocks, float * out0, float * out1) { - const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; - const uint64_t gather_0_to_7 = 0x398a418820ULL; + const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; __asm__ volatile("flw.ps f31, %[g]\n" : : [g] "m"(*(const int32_t (*)[8]) gather_pattern) : "f31"); __asm__ volatile( "fbci.pi f20, 0\n" @@ -371,225 +288,83 @@ static inline void q8_dot_compute_x2_aligned(const block_q8_0 * q_row0, const block_q8_0 * blk1 = q_row1 + kb; const float * b_ptr = b_col + (kb << 5); - const uintptr_t qs_addr0 = (uintptr_t) blk0->qs; - const uintptr_t qs_addr1 = (uintptr_t) blk1->qs; - const uintptr_t qs_aligned0 = qs_addr0 & ~(uintptr_t) 31; - const uintptr_t qs_aligned1 = qs_addr1 & ~(uintptr_t) 31; - const int fast_chunks = (int) ((32 - (qs_addr0 & 31)) >> 3); - - if (fast_chunks >= 3) { - __asm__ volatile( - "fbci.pi f10, 0\n" - "fbci.pi f11, 0\n" - - "flw.ps f12, %[bv0]\n" - "fg32b.ps f16, %[gi](%[r0ap0])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f12, f10\n" - "fg32b.ps f17, %[gi](%[r1ap0])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f12, f11\n" - - "flw.ps f13, %[bv1]\n" - "fg32b.ps f16, %[gi](%[r0ap1])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f13, f10\n" - "fg32b.ps f17, %[gi](%[r1ap1])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f13, f11\n" - - "flw.ps f14, %[bv2]\n" - "fg32b.ps f16, %[gi](%[r0ap2])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f14, f10\n" - "fg32b.ps f17, %[gi](%[r1ap2])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f14, f11\n" - - "flw.ps f15, %[bv3]\n" - "fgb.ps f16, f31(%[r0ap3])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f15, f10\n" - "fgb.ps f17, f31(%[r1ap3])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f15, f11\n" - : - : [gi] "r"(gather_0_to_7), [r0ap0] "r"(qs_addr0), [r0ap1] "r"(qs_aligned0 | ((qs_addr0 + 8) & 31)), - [r0ap2] "r"(qs_aligned0 | ((qs_addr0 + 16) & 31)), [r0ap3] "r"(&blk0->qs[24]), [r1ap0] "r"(qs_addr1), - [r1ap1] "r"(qs_aligned1 | ((qs_addr1 + 8) & 31)), [r1ap2] "r"(qs_aligned1 | ((qs_addr1 + 16) & 31)), - [r1ap3] "r"(&blk1->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), - [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), - [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); - } else if (fast_chunks == 2) { - __asm__ volatile( - "fbci.pi f10, 0\n" - "fbci.pi f11, 0\n" - - "flw.ps f12, %[bv0]\n" - "fg32b.ps f16, %[gi](%[r0ap0])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f12, f10\n" - "fg32b.ps f17, %[gi](%[r1ap0])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f12, f11\n" - - "flw.ps f13, %[bv1]\n" - "fg32b.ps f16, %[gi](%[r0ap1])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f13, f10\n" - "fg32b.ps f17, %[gi](%[r1ap1])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f13, f11\n" - - "flw.ps f14, %[bv2]\n" - "fgb.ps f16, f31(%[r0ap2])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f14, f10\n" - "fgb.ps f17, f31(%[r1ap2])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f14, f11\n" - - "flw.ps f15, %[bv3]\n" - "fgb.ps f16, f31(%[r0ap3])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f15, f10\n" - "fgb.ps f17, f31(%[r1ap3])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f15, f11\n" - : - : [gi] "r"(gather_0_to_7), [r0ap0] "r"(qs_addr0), [r0ap1] "r"(qs_aligned0 | ((qs_addr0 + 8) & 31)), - [r0ap2] "r"(&blk0->qs[16]), [r0ap3] "r"(&blk0->qs[24]), [r1ap0] "r"(qs_addr1), - [r1ap1] "r"(qs_aligned1 | ((qs_addr1 + 8) & 31)), [r1ap2] "r"(&blk1->qs[16]), - [r1ap3] "r"(&blk1->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), - [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), - [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); - } else if (fast_chunks == 1) { - __asm__ volatile( - "fbci.pi f10, 0\n" - "fbci.pi f11, 0\n" - - "flw.ps f12, %[bv0]\n" - "fg32b.ps f16, %[gi](%[r0ap0])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f12, f10\n" - "fg32b.ps f17, %[gi](%[r1ap0])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f12, f11\n" - - "flw.ps f13, %[bv1]\n" - "fgb.ps f16, f31(%[r0ap1])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f13, f10\n" - "fgb.ps f17, f31(%[r1ap1])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f13, f11\n" - - "flw.ps f14, %[bv2]\n" - "fgb.ps f16, f31(%[r0ap2])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f14, f10\n" - "fgb.ps f17, f31(%[r1ap2])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f14, f11\n" - - "flw.ps f15, %[bv3]\n" - "fgb.ps f16, f31(%[r0ap3])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f15, f10\n" - "fgb.ps f17, f31(%[r1ap3])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f15, f11\n" - : - : [gi] "r"(gather_0_to_7), [r0ap0] "r"(qs_addr0), [r0ap1] "r"(&blk0->qs[8]), [r0ap2] "r"(&blk0->qs[16]), - [r0ap3] "r"(&blk0->qs[24]), [r1ap0] "r"(qs_addr1), [r1ap1] "r"(&blk1->qs[8]), - [r1ap2] "r"(&blk1->qs[16]), [r1ap3] "r"(&blk1->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), - [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), - [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); - } else { - __asm__ volatile( - "fbci.pi f10, 0\n" - "fbci.pi f11, 0\n" - - "flw.ps f12, %[bv0]\n" - "fgb.ps f16, f31(%[r0ap0])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f12, f10\n" - "fgb.ps f17, f31(%[r1ap0])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f12, f11\n" - - "flw.ps f13, %[bv1]\n" - "fgb.ps f16, f31(%[r0ap1])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f13, f10\n" - "fgb.ps f17, f31(%[r1ap1])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f13, f11\n" - - "flw.ps f14, %[bv2]\n" - "fgb.ps f16, f31(%[r0ap2])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f14, f10\n" - "fgb.ps f17, f31(%[r1ap2])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f14, f11\n" - - "flw.ps f15, %[bv3]\n" - "fgb.ps f16, f31(%[r0ap3])\n" - "fcvt.ps.pw f16, f16\n" - "fmadd.ps f10, f16, f15, f10\n" - "fgb.ps f17, f31(%[r1ap3])\n" - "fcvt.ps.pw f17, f17\n" - "fmadd.ps f11, f17, f15, f11\n" - : - : [r0ap0] "r"(&blk0->qs[0]), [r0ap1] "r"(&blk0->qs[8]), [r0ap2] "r"(&blk0->qs[16]), - [r0ap3] "r"(&blk0->qs[24]), [r1ap0] "r"(&blk1->qs[0]), [r1ap1] "r"(&blk1->qs[8]), - [r1ap2] "r"(&blk1->qs[16]), [r1ap3] "r"(&blk1->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), - [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), - [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) - : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); - } - - const uint32_t scale_raw0 = (uint32_t) blk0->d; - const uint32_t scale_raw1 = (uint32_t) blk1->d; __asm__ volatile( - "fbcx.ps f24, %[s0]\n" - "fcvt.ps.f16 f24, f24\n" - "fmadd.ps f20, f10, f24, f20\n" - "fbcx.ps f25, %[s1]\n" - "fcvt.ps.f16 f25, f25\n" - "fmadd.ps f21, f11, f25, f21\n" + "fbci.pi f10, 0\n" + "fbci.pi f11, 0\n" + + "flw.ps f12, %[bv0]\n" + "fgb.ps f16, f31(%[r0ap0])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f12, f10\n" + "fgb.ps f17, f31(%[r1ap0])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f12, f11\n" + + "flw.ps f13, %[bv1]\n" + "fgb.ps f16, f31(%[r0ap1])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f13, f10\n" + "fgb.ps f17, f31(%[r1ap1])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f13, f11\n" + + "flw.ps f14, %[bv2]\n" + "fgb.ps f16, f31(%[r0ap2])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f14, f10\n" + "fgb.ps f17, f31(%[r1ap2])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f14, f11\n" + + "flw.ps f15, %[bv3]\n" + "fgb.ps f16, f31(%[r0ap3])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f15, f10\n" + "fgb.ps f17, f31(%[r1ap3])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f15, f11\n" + : + : [r0ap0] "r"(&blk0->qs[0]), [r0ap1] "r"(&blk0->qs[8]), [r0ap2] "r"(&blk0->qs[16]), [r0ap3] "r"(&blk0->qs[24]), + [r1ap0] "r"(&blk1->qs[0]), [r1ap1] "r"(&blk1->qs[8]), [r1ap2] "r"(&blk1->qs[16]), [r1ap3] "r"(&blk1->qs[24]), + [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), + [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); + + // f20 += f10 * broadcast(scale0); f21 += f11 * broadcast(scale1) + uint32_t scale_raw0 = (uint32_t) blk0->d; + uint32_t scale_raw1 = (uint32_t) blk1->d; + __asm__ volatile( + "fbcx.ps f18, %[s0]\n" + "fbcx.ps f19, %[s1]\n" + "fcvt.ps.f16 f18, f18\n" + "fcvt.ps.f16 f19, f19\n" + "fmadd.ps f20, f10, f18, f20\n" + "fmadd.ps f21, f11, f19, f21\n" : : [s0] "r"(scale_raw0), [s1] "r"(scale_raw1) - : "f20", "f21", "f24", "f25"); + : "f18", "f19", "f20", "f21"); } - - float result0; - float result1; - __asm__ __volatile__( + float s0, s1; + __asm__ volatile( "fswizz.ps f1, f20, 0xB1 \n\t" "fadd.ps f2, f20, f1, rne \n\t" "fswizz.ps f3, f2, 0x4E \n\t" "fadd.ps f4, f2, f3, rne \n\t" "fmvz.x.ps t0, f4, 4 \n\t" "fbcx.ps f5, t0 \n\t" - "fadd.ps %[vout], f4, f5, rne \n\t" - : [vout] "=f"(result0)::"t0", "f1", "f2", "f3", "f4", "f5"); - __asm__ __volatile__( + "fadd.ps %[v0], f4, f5, rne \n\t" "fswizz.ps f1, f21, 0xB1 \n\t" "fadd.ps f2, f21, f1, rne \n\t" "fswizz.ps f3, f2, 0x4E \n\t" "fadd.ps f4, f2, f3, rne \n\t" "fmvz.x.ps t0, f4, 4 \n\t" "fbcx.ps f5, t0 \n\t" - "fadd.ps %[vout], f4, f5, rne \n\t" - : [vout] "=f"(result1)::"t0", "f1", "f2", "f3", "f4", "f5"); - - *out0 = result0; - *out1 = result1; + "fadd.ps %[v1], f4, f5, rne \n\t" + : [v0] "=f"(s0), [v1] "=f"(s1) + : + : "t0", "f1", "f2", "f3", "f4", "f5", "f20", "f21"); + *out0 = s0; + *out1 = s1; } // Compute dot product between f16 block and f32 column vector (NAIVE VERSION) @@ -882,6 +657,7 @@ static inline void q4_dot_compute_x2_aligned(const block_q4_0 * q_row0, float * out1) { const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; __asm__ volatile("flw.ps f31, %[g]\n" : : [g] "m"(*(const int32_t (*)[8]) gather_pattern) : "f31"); + __asm__ volatile( "fbci.pi f20, 0\n" "fbci.pi f21, 0\n" :: @@ -995,3 +771,531 @@ static inline void q4_dot_compute_x2_aligned(const block_q4_0 * q_row0, *out0 = result0; *out1 = result1; } + + +// Full-row dot product for Q4_K weights against an F32 activation column. +// +// Unlike Q4_0/Q8_0 (whose dequant is a pure per-block scale, so the scale can +// be factored out of the dot product), Q4_K reconstructs each weight via an +// affine transform `w = d*scale*q - dmin*min` with per-group scales/mins inside +// each 256-element super-block. That makes the cheap "scale the integer dot" +// trick inapplicable. +// +// The dequant math mirrors dequantize_q4_K_block exactly, but the per-element +// product is folded straight into a scalar accumulator instead of being staged +// through a temporary buffer. This deliberately avoids a large (1KB) on-stack +// dequant buffer and the vector-mask save/restore of the F32 dot helper, both +// of which are unsafe in the uberkernel context. +// +// K_sblocks is the number of QK_K (256) element super-blocks in the row +// (i.e. K / QK_K). +static inline float sw_fp16_to_fp32(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000) << 16; + uint32_t exp = (h >> 10) & 0x1F; + uint32_t mant = h & 0x3FF; + uint32_t f; + if (exp == 0) { + if (mant == 0) { f = sign; } + else { + exp = 127 - 15 + 1; + while ((mant & 0x400) == 0) { mant <<= 1; exp--; } + mant &= 0x3FF; + f = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1F) { + f = sign | 0x7F800000u | (mant << 13); + } else { + f = sign | ((exp + (127 - 15)) << 23) | (mant << 13); + } + float out; __builtin_memcpy(&out, &f, 4); return out; +} + +static inline float compute_row_dot_q4_K(const block_q4_K* q_row, + const float* b_col, + int64_t K_sblocks) { + float acc = 0.0f; + for (int64_t sb = 0; sb < K_sblocks; sb++) { + const block_q4_K* block = q_row + sb; + const float* b = b_col + sb * QK_K; + const uint8_t* q = block->qs; + const float d = sw_fp16_to_fp32(block->d); + const float min = sw_fp16_to_fp32(block->dmin); + + int is = 0; + uint8_t sc, m; + for (int j = 0; j < QK_K; j += 64) { + get_scale_min_k4(is + 0, block->scales, &sc, &m); + const float d1 = d * sc; + const float m1 = min * m; + get_scale_min_k4(is + 1, block->scales, &sc, &m); + const float d2 = d * sc; + const float m2 = min * m; + for (int l = 0; l < 32; ++l) { + acc += (d1 * (float)(q[l] & 0xF) - m1) * (*b++); + } + for (int l = 0; l < 32; ++l) { + acc += (d2 * (float)(q[l] >> 4) - m2) * (*b++); + } + q += 32; + is += 2; + } + } + return acc; +} + +// Vectorized (8-wide) full-row dot for Q4_K. Affine 8-groups-of-32 layout +// w = d*sc*nibble - dmin*m (no qh bit). Group g -> qs pair g/2, nibble +// low(g even)/high(g odd). f10 scale term, f9 min term; result = f10 - f9. +#define Q4V_LO "fandi.pi f12, f11, 15\n\t" +#define Q4V_HI "fsrli.pi f12, f11, 4\n\t fandi.pi f12, f12, 15\n\t" +#define Q4V_CHUNK(NIB, qlp, bp, dscb, dmb) \ + __asm__ volatile( \ + "fgb.ps f11, f31(%[q])\n\t" \ + NIB \ + "fcvt.ps.pw f12, f12, rne\n\t" \ + "fbcx.ps f16, %[dsc]\n\t" \ + "flw.ps f15, 0(%[b])\n\t" \ + "fmul.ps f12, f12, f16\n\t" \ + "fmadd.ps f10, f12, f15, f10\n\t" \ + "fbcx.ps f17, %[dm]\n\t" \ + "fmadd.ps f9, f15, f17, f9\n\t" \ + :: [q] "r"(qlp), [b] "r"(bp), [dsc] "r"(dscb), [dm] "r"(dmb) \ + : "f11", "f12", "f15", "f16", "f17") +#define Q4V_GROUP(NIB, pp) \ + do { \ + const uint8_t* qlb = block->qs + (pp) * 32; \ + Q4V_CHUNK(NIB, qlb + 0, bg + 0, dscb, dmb); \ + Q4V_CHUNK(NIB, qlb + 8, bg + 8, dscb, dmb); \ + Q4V_CHUNK(NIB, qlb + 16, bg + 16, dscb, dmb); \ + Q4V_CHUNK(NIB, qlb + 24, bg + 24, dscb, dmb); \ + } while (0) + +static inline float compute_row_dot_q4_K_vec(const block_q4_K* q_row, + const float* b_col, + int64_t K_sblocks) { + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + static const int32_t __attribute__((aligned(64))) gp[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + __asm__ volatile("flw.ps f31, %[g]\n\t" + "fbci.ps f10, 0\n\t" + "fbci.ps f9, 0\n\t" + :: [g] "m"(*(const int32_t(*)[8]) gp) + : "f31", "f10", "f9"); + + for (int64_t sb = 0; sb < K_sblocks; sb++) { + const block_q4_K* block = q_row + sb; + const float* b = b_col + sb * QK_K; + const float d = sw_fp16_to_fp32(block->d); + const float min = sw_fp16_to_fp32(block->dmin); + + for (int g = 0; g < 8; ++g) { + uint8_t sc, m; + get_scale_min_k4(g, block->scales, &sc, &m); + const float dscf = d * (float) sc; + const float dmf = min * (float) m; + uint32_t dscb, dmb; + __builtin_memcpy(&dscb, &dscf, 4); + __builtin_memcpy(&dmb, &dmf, 4); + const float* bg = b + g * 32; + const int p = g >> 1; + + if (g & 1) { + Q4V_GROUP(Q4V_HI, p); + } else { + Q4V_GROUP(Q4V_LO, p); + } + } + } + + float final_sum; + __asm__ volatile( + "fswizz.ps f1, f10, 0xB1 \n\t fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps f6, f4, f5, rne \n\t" + "fswizz.ps f1, f9, 0xB1 \n\t fadd.ps f2, f9, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps f7, f4, f5, rne \n\t" + "fsub.ps %[out], f6, f7, rne \n\t" + : [out] "=f"(final_sum) + :: "t0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f9", "f10"); + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return final_sum; +} + +// Vectorized (8-wide) full-row dot for Q6_K, modeled on the q4_0/q8_0 vector +// dots. Each 32-element group of contiguous output elements maps to contiguous +// ql/qh bytes and a fixed nibble/shift, so a group is 4 chunks of 8 lanes. +// The per-group factor d*scale is folded into the weight vector (fmul), so a +// single fp32 accumulator (f10) covers the whole row and is reduced once at the +// end. f30 holds -32.0 (the Q6_K zero point), f31 the byte-gather pattern. +#define Q6V_NIB_LO "fandi.pi f12, f11, 15\n\t" +#define Q6V_NIB_HI "fsrli.pi f12, f11, 4\n\t fandi.pi f12, f12, 15\n\t" +#define Q6V_CHUNK(NIB, SH, qlp, qhp, bp, facbits) \ + __asm__ volatile( \ + "fgb.ps f11, f31(%[q])\n\t" \ + "fgb.ps f13, f31(%[h])\n\t" \ + NIB \ + "fsrli.pi f13, f13, " #SH "\n\t" \ + "fandi.pi f13, f13, 3\n\t" \ + "fslli.pi f13, f13, 4\n\t" \ + "fcvt.ps.pw f12, f12, rne\n\t" \ + "fcvt.ps.pw f13, f13, rne\n\t" \ + "fadd.ps f12, f12, f13, rne\n\t" \ + "fadd.ps f12, f12, f30, rne\n\t" \ + "fbcx.ps f16, %[f]\n\t" \ + "flw.ps f15, 0(%[b])\n\t" \ + "fmul.ps f12, f12, f16\n\t" \ + "fmadd.ps f10, f12, f15, f10\n\t" \ + :: [q] "r"(qlp), [h] "r"(qhp), [b] "r"(bp), [f] "r"(facbits) \ + : "f11", "f12", "f13", "f15", "f16") + +static inline float compute_row_dot_q6_K_vec(const block_q6_K* q_row, + const float* b_col, + int64_t K_sblocks) { + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + static const int32_t __attribute__((aligned(64))) gp[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + const uint32_t neg32 = 0xC2000000u; // -32.0f + __asm__ volatile("flw.ps f31, %[g]\n\t" + "fbcx.ps f30, %[n]\n\t" + "fbci.ps f10, 0\n\t" + :: [g] "m"(*(const int32_t(*)[8]) gp), [n] "r"(neg32) + : "f31", "f30", "f10"); + + for (int64_t sb = 0; sb < K_sblocks; sb++) { + const block_q6_K* block = q_row + sb; + const float* b = b_col + sb * QK_K; + const float d = sw_fp16_to_fp32(block->d); + + for (int g = 0; g < 8; ++g) { + const int c = g >> 2; + const int sub = g & 3; + const uint8_t* qlbase = block->ql + 64 * c + (sub & 1) * 32; + const uint8_t* qhbase = block->qh + 32 * c; + const float* bg = b + g * 32; + const float f0f = d * (float) block->scales[8 * c + 2 * sub + 0]; + const float f1f = d * (float) block->scales[8 * c + 2 * sub + 1]; + uint32_t f0, f1; + __builtin_memcpy(&f0, &f0f, 4); + __builtin_memcpy(&f1, &f1f, 4); + + switch (sub) { + case 0: + Q6V_CHUNK(Q6V_NIB_LO, 0, qlbase + 0, qhbase + 0, bg + 0, f0); + Q6V_CHUNK(Q6V_NIB_LO, 0, qlbase + 8, qhbase + 8, bg + 8, f0); + Q6V_CHUNK(Q6V_NIB_LO, 0, qlbase + 16, qhbase + 16, bg + 16, f1); + Q6V_CHUNK(Q6V_NIB_LO, 0, qlbase + 24, qhbase + 24, bg + 24, f1); + break; + case 1: + Q6V_CHUNK(Q6V_NIB_LO, 2, qlbase + 0, qhbase + 0, bg + 0, f0); + Q6V_CHUNK(Q6V_NIB_LO, 2, qlbase + 8, qhbase + 8, bg + 8, f0); + Q6V_CHUNK(Q6V_NIB_LO, 2, qlbase + 16, qhbase + 16, bg + 16, f1); + Q6V_CHUNK(Q6V_NIB_LO, 2, qlbase + 24, qhbase + 24, bg + 24, f1); + break; + case 2: + Q6V_CHUNK(Q6V_NIB_HI, 4, qlbase + 0, qhbase + 0, bg + 0, f0); + Q6V_CHUNK(Q6V_NIB_HI, 4, qlbase + 8, qhbase + 8, bg + 8, f0); + Q6V_CHUNK(Q6V_NIB_HI, 4, qlbase + 16, qhbase + 16, bg + 16, f1); + Q6V_CHUNK(Q6V_NIB_HI, 4, qlbase + 24, qhbase + 24, bg + 24, f1); + break; + default: + Q6V_CHUNK(Q6V_NIB_HI, 6, qlbase + 0, qhbase + 0, bg + 0, f0); + Q6V_CHUNK(Q6V_NIB_HI, 6, qlbase + 8, qhbase + 8, bg + 8, f0); + Q6V_CHUNK(Q6V_NIB_HI, 6, qlbase + 16, qhbase + 16, bg + 16, f1); + Q6V_CHUNK(Q6V_NIB_HI, 6, qlbase + 24, qhbase + 24, bg + 24, f1); + break; + } + } + } + + float final_sum; + __asm__ volatile( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[out], f4, f5, rne \n\t" + : [out] "=f"(final_sum) + :: "t0", "f1", "f2", "f3", "f4", "f5", "f10"); + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return final_sum; +} + +// NOTE: two further Q6_K dot experiments were tried and REMOVED as neither +// helped - the generation dot is memory/overhead bound, not compute bound: +// * gather-reduced (ql/qh gathered once, reused across sub-groups, 64->24 +// fgb/super-block) - perf-neutral (14.95 vs 14.93 t/s). +// * 4 rotating accumulators to pipeline the fmadd chain - also neutral (15.02). + +// Vectorized (8-wide) full-row dot for Q5_K. Same 8-groups-of-32 affine layout +// as Q4_K plus a 5th bit from qh: w = d*sc*(nibble + qh_bit*16) - dmin*m. +// Group g -> qs pair g/2, nibble low(g even)/high(g odd), qh bit at position g. +// f10 accumulates the scale term, f9 the min term (sum of b*dmin*m per group); +// result = reduce(f10) - reduce(f9). +#define Q5V_LO "fandi.pi f12, f11, 15\n\t" +#define Q5V_HI "fsrli.pi f12, f11, 4\n\t fandi.pi f12, f12, 15\n\t" +#define Q5V_CHUNK(NIB, BP, qlp, qhp, bp, dscb, dmb) \ + __asm__ volatile( \ + "fgb.ps f11, f31(%[q])\n\t" \ + "fgb.ps f13, f31(%[h])\n\t" \ + NIB \ + "fsrli.pi f13, f13, " #BP "\n\t" \ + "fandi.pi f13, f13, 1\n\t" \ + "fslli.pi f13, f13, 4\n\t" \ + "fcvt.ps.pw f12, f12, rne\n\t" \ + "fcvt.ps.pw f13, f13, rne\n\t" \ + "fadd.ps f12, f12, f13, rne\n\t" \ + "fbcx.ps f16, %[dsc]\n\t" \ + "flw.ps f15, 0(%[b])\n\t" \ + "fmul.ps f12, f12, f16\n\t" \ + "fmadd.ps f10, f12, f15, f10\n\t" \ + "fbcx.ps f17, %[dm]\n\t" \ + "fmadd.ps f9, f15, f17, f9\n\t" \ + :: [q] "r"(qlp), [h] "r"(qhp), [b] "r"(bp), [dsc] "r"(dscb), [dm] "r"(dmb) \ + : "f11", "f12", "f13", "f15", "f16", "f17") +#define Q5V_GROUP(NIB, BP, pp) \ + do { \ + const uint8_t* qlb = block->qs + (pp) * 32; \ + const uint8_t* qhb = block->qh; \ + Q5V_CHUNK(NIB, BP, qlb + 0, qhb + 0, bg + 0, dscb, dmb); \ + Q5V_CHUNK(NIB, BP, qlb + 8, qhb + 8, bg + 8, dscb, dmb); \ + Q5V_CHUNK(NIB, BP, qlb + 16, qhb + 16, bg + 16, dscb, dmb); \ + Q5V_CHUNK(NIB, BP, qlb + 24, qhb + 24, bg + 24, dscb, dmb); \ + } while (0) + +static inline float compute_row_dot_q5_K_vec(const block_q5_K* q_row, + const float* b_col, + int64_t K_sblocks) { + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + static const int32_t __attribute__((aligned(64))) gp[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + __asm__ volatile("flw.ps f31, %[g]\n\t" + "fbci.ps f10, 0\n\t" + "fbci.ps f9, 0\n\t" + :: [g] "m"(*(const int32_t(*)[8]) gp) + : "f31", "f10", "f9"); + + for (int64_t sb = 0; sb < K_sblocks; sb++) { + const block_q5_K* block = q_row + sb; + const float* b = b_col + sb * QK_K; + const float d = sw_fp16_to_fp32(block->d); + const float min = sw_fp16_to_fp32(block->dmin); + + for (int g = 0; g < 8; ++g) { + uint8_t sc, m; + get_scale_min_k4(g, block->scales, &sc, &m); + const float dscf = d * (float) sc; + const float dmf = min * (float) m; + uint32_t dscb, dmb; + __builtin_memcpy(&dscb, &dscf, 4); + __builtin_memcpy(&dmb, &dmf, 4); + const float* bg = b + g * 32; + const int p = g >> 1; + + switch (g) { + case 0: Q5V_GROUP(Q5V_LO, 0, p); break; + case 1: Q5V_GROUP(Q5V_HI, 1, p); break; + case 2: Q5V_GROUP(Q5V_LO, 2, p); break; + case 3: Q5V_GROUP(Q5V_HI, 3, p); break; + case 4: Q5V_GROUP(Q5V_LO, 4, p); break; + case 5: Q5V_GROUP(Q5V_HI, 5, p); break; + case 6: Q5V_GROUP(Q5V_LO, 6, p); break; + default: Q5V_GROUP(Q5V_HI, 7, p); break; + } + } + } + + float final_sum; + __asm__ volatile( + "fswizz.ps f1, f10, 0xB1 \n\t fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps f6, f4, f5, rne \n\t" + "fswizz.ps f1, f9, 0xB1 \n\t fadd.ps f2, f9, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps f7, f4, f5, rne \n\t" + "fsub.ps %[out], f6, f7, rne \n\t" + : [out] "=f"(final_sum) + :: "t0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f9", "f10"); + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return final_sum; +} + +// Vectorized (8-wide) full-row dot for Q2_K. Affine 2-bit +// w = d*(sc&0xF)*q - dmin*(sc>>4), 16 groups of 16. Sub-group sg -> chunk +// c=sg/8, shift 2*((sg&7)/2), A/B nibble-set ab=(sg&7)&1 selecting qs/scale. +// f10 scale term, f9 min term; result = reduce(f10) - reduce(f9). +#define Q2V_CHUNK(SH, qp, bp, dlb, mlb) \ + __asm__ volatile( \ + "fgb.ps f11, f31(%[q])\n\t" \ + "fsrli.pi f12, f11, " #SH "\n\t" \ + "fandi.pi f12, f12, 3\n\t" \ + "fcvt.ps.pw f12, f12, rne\n\t" \ + "fbcx.ps f16, %[dl]\n\t" \ + "flw.ps f15, 0(%[b])\n\t" \ + "fmul.ps f12, f12, f16\n\t" \ + "fmadd.ps f10, f12, f15, f10\n\t" \ + "fbcx.ps f17, %[ml]\n\t" \ + "fmadd.ps f9, f15, f17, f9\n\t" \ + :: [q] "r"(qp), [b] "r"(bp), [dl] "r"(dlb), [ml] "r"(mlb) \ + : "f11", "f12", "f15", "f16", "f17") +#define Q2V_GROUP(SH, qb, bg) \ + do { \ + Q2V_CHUNK(SH, (qb) + 0, (bg) + 0, dlb, mlb); \ + Q2V_CHUNK(SH, (qb) + 8, (bg) + 8, dlb, mlb); \ + } while (0) + +static inline float compute_row_dot_q2_K_vec(const block_q2_K* q_row, + const float* b_col, + int64_t K_sblocks) { + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + static const int32_t __attribute__((aligned(64))) gp[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + __asm__ volatile("flw.ps f31, %[g]\n\t" + "fbci.ps f10, 0\n\t" + "fbci.ps f9, 0\n\t" + :: [g] "m"(*(const int32_t(*)[8]) gp) + : "f31", "f10", "f9"); + + for (int64_t sb = 0; sb < K_sblocks; sb++) { + const block_q2_K* block = q_row + sb; + const float* b = b_col + sb * QK_K; + const float d = sw_fp16_to_fp32(block->d); + const float min = sw_fp16_to_fp32(block->dmin); + + for (int sg = 0; sg < 16; ++sg) { + const int c = sg >> 3; + const int sgi = sg & 7; + const int j = sgi >> 1; + const int ab = sgi & 1; + const uint8_t sc = block->scales[c * 8 + 2 * j + ab]; + const float dlf = d * (float) (sc & 0xF); + const float mlf = min * (float) (sc >> 4); + uint32_t dlb, mlb; + __builtin_memcpy(&dlb, &dlf, 4); + __builtin_memcpy(&mlb, &mlf, 4); + const uint8_t* qb = block->qs + c * 32 + ab * 16; + const float* bg = b + sg * 16; + + switch (j) { + case 0: Q2V_GROUP(0, qb, bg); break; + case 1: Q2V_GROUP(2, qb, bg); break; + case 2: Q2V_GROUP(4, qb, bg); break; + default: Q2V_GROUP(6, qb, bg); break; + } + } + } + + float final_sum; + __asm__ volatile( + "fswizz.ps f1, f10, 0xB1 \n\t fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps f6, f4, f5, rne \n\t" + "fswizz.ps f1, f9, 0xB1 \n\t fadd.ps f2, f9, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps f7, f4, f5, rne \n\t" + "fsub.ps %[out], f6, f7, rne \n\t" + : [out] "=f"(final_sum) + :: "t0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f9", "f10"); + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return final_sum; +} + +// Vectorized (8-wide) full-row dot for Q3_K. 3-bit +// w = d*(scale-32)*(q2 + hbit*4 - 4), 16 groups of 16, no min term. Sub-group +// sg -> c=sg/8, j=(sg&7)/2 (qs shift 2*j), ab=(sg&7)&1 (qs/hmask offset), +// hmask bit at position c*4+j. f30 holds -4.0. +#define Q3V_CHUNK(SH, BP, qp, hp, bp, dlb) \ + __asm__ volatile( \ + "fgb.ps f11, f31(%[q])\n\t" \ + "fgb.ps f13, f31(%[h])\n\t" \ + "fsrli.pi f12, f11, " #SH "\n\t fandi.pi f12, f12, 3\n\t" \ + "fsrli.pi f13, f13, " #BP "\n\t fandi.pi f13, f13, 1\n\t fslli.pi f13, f13, 2\n\t" \ + "fcvt.ps.pw f12, f12, rne\n\t" \ + "fcvt.ps.pw f13, f13, rne\n\t" \ + "fadd.ps f12, f12, f13, rne\n\t" \ + "fadd.ps f12, f12, f30, rne\n\t" \ + "fbcx.ps f16, %[dl]\n\t" \ + "flw.ps f15, 0(%[b])\n\t" \ + "fmul.ps f12, f12, f16\n\t" \ + "fmadd.ps f10, f12, f15, f10\n\t" \ + :: [q] "r"(qp), [h] "r"(hp), [b] "r"(bp), [dl] "r"(dlb) \ + : "f11", "f12", "f13", "f15", "f16") +#define Q3V_GROUP(SH, BP, qb, hb, bg) \ + do { \ + Q3V_CHUNK(SH, BP, (qb) + 0, (hb) + 0, (bg) + 0, dlb); \ + Q3V_CHUNK(SH, BP, (qb) + 8, (hb) + 8, (bg) + 8, dlb); \ + } while (0) + +static inline float compute_row_dot_q3_K_vec(const block_q3_K* q_row, + const float* b_col, + int64_t K_sblocks) { + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + static const int32_t __attribute__((aligned(64))) gp[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + const uint32_t neg4 = 0xC0800000u; // -4.0f + __asm__ volatile("flw.ps f31, %[g]\n\t" + "fbcx.ps f30, %[n]\n\t" + "fbci.ps f10, 0\n\t" + :: [g] "m"(*(const int32_t(*)[8]) gp), [n] "r"(neg4) + : "f31", "f30", "f10"); + + for (int64_t sb = 0; sb < K_sblocks; sb++) { + const block_q3_K* block = q_row + sb; + const float* b = b_col + sb * QK_K; + const float d = sw_fp16_to_fp32(block->d); + int8_t scales[16]; + unpack_q3_K_scales(block->scales, scales); + + for (int sg = 0; sg < 16; ++sg) { + const int c = sg >> 3; + const int sgi = sg & 7; + const int j = sgi >> 1; + const int ab = sgi & 1; + const float dlf = d * (float) (scales[c * 8 + 2 * j + ab] - 32); + uint32_t dlb; + __builtin_memcpy(&dlb, &dlf, 4); + const uint8_t* qb = block->qs + c * 32 + ab * 16; + const uint8_t* hb = block->hmask + ab * 16; + const float* bg = b + sg * 16; + const int cj = c * 4 + j; // hmask bit position + + switch (cj) { + case 0: Q3V_GROUP(0, 0, qb, hb, bg); break; + case 1: Q3V_GROUP(2, 1, qb, hb, bg); break; + case 2: Q3V_GROUP(4, 2, qb, hb, bg); break; + case 3: Q3V_GROUP(6, 3, qb, hb, bg); break; + case 4: Q3V_GROUP(0, 4, qb, hb, bg); break; + case 5: Q3V_GROUP(2, 5, qb, hb, bg); break; + case 6: Q3V_GROUP(4, 6, qb, hb, bg); break; + default: Q3V_GROUP(6, 7, qb, hb, bg); break; + } + } + } + + float final_sum; + __asm__ volatile( + "fswizz.ps f1, f10, 0xB1 \n\t fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t fbcx.ps f5, t0 \n\t fadd.ps %[out], f4, f5, rne \n\t" + : [out] "=f"(final_sum) + :: "t0", "f1", "f2", "f3", "f4", "f5", "f10"); + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return final_sum; +} diff --git a/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c b/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c index c905b366f380..7c93ad778518 100644 --- a/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +++ b/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c @@ -358,9 +358,12 @@ int entry_point(struct ggml_et_flash_attn_ext_params * params, void * env) { const int is_hart1 = hart_id & 1; uint64_t local_minion = (hart_id >> 1) & 0x1F; - struct ggml_tensor * q = ¶ms->src0; - struct ggml_tensor * k = ¶ms->src1; - struct ggml_tensor * v = ¶ms->src2; + struct ggml_tensor * q = ¶ms->src0; + evict_region_past_l2(q->data, tensor_bytes_fa(q)); + struct ggml_tensor * k = ¶ms->src1; + evict_region_past_l2(k->data, tensor_bytes_fa(k)); + struct ggml_tensor * v = ¶ms->src2; + evict_region_past_l2(v->data, tensor_bytes_fa(v)); struct ggml_tensor * dst = ¶ms->dst; const int32_t has_mask = params->has_mask; struct ggml_tensor * mask = has_mask ? ¶ms->mask : (struct ggml_tensor *) 0; @@ -371,13 +374,13 @@ int entry_point(struct ggml_et_flash_attn_ext_params * params, void * env) { char * dst_data = (char *) dst->data; // et_barrier(ET_BARRIER_GLOBAL); - evict_region_past_l2(q->data, tensor_bytes_fa(q)); - evict_region_past_l2(k->data, tensor_bytes_fa(k)); - evict_region_past_l2(v->data, tensor_bytes_fa(v)); - if (mask) { - evict_region_past_l2(mask->data, tensor_bytes_fa(mask)); - } - et_barrier(ET_BARRIER_GLOBAL); + // evict_region_past_l2(q_data, tensor_bytes_fa(q)); + // evict_region_past_l2(k_data, tensor_bytes_fa(k)); + // evict_region_past_l2(v_data, tensor_bytes_fa(v)); + // if (mask) { + // evict_region_past_l2(mask->data, tensor_bytes_fa(mask)); + // } + // et_barrier(ET_BARRIER_GLOBAL); const int64_t dk = q->ne[0]; const int64_t nq = q->ne[1]; @@ -462,9 +465,14 @@ int entry_point(struct ggml_et_flash_attn_ext_params * params, void * env) { // All teams in a shire must iterate the same number of times so the // per-iter shire barriers stay balanced. Teams whose assigned row is // past total_rows still call the barriers but skip the packing work. + et_barrier(ET_BARRIER_SHIRE); // et_barrier(ET_BARRIER_GLOBAL); + if (is_hart1) { + // et_barrier(ET_BARRIER_GLOBAL); + // et_barrier(ET_BARRIER_SHIRE); + uint32_t chunk_id = 0; const int64_t row_base = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; @@ -548,10 +556,10 @@ int entry_point(struct ggml_et_flash_attn_ext_params * params, void * env) { } // Hart 0: tensor engine compute -#ifndef UBERKERNEL_SUPPRESS_SCP_SETUP - setup_cache_scp(); -#endif - CLEAR_TENSOR_ERROR; + // #ifndef UBERKERNEL_SUPPRESS_SCP_SETUP + // setup_cache_scp(); + // #endif + // CLEAR_TENSOR_ERROR; // Q converted to F16 (one row at a time) et_fp16_t q_f16[FA_DK_MAX] __attribute__((aligned(64))); diff --git a/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c b/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c index 701f1db98e0e..db69ebba2885 100644 --- a/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +++ b/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c @@ -137,6 +137,70 @@ static void copy_q4_K_row(float * dst, const block_q4_K * src_blocks, int64_t nu } } +// Copy a row of Q2_K data to F32 destination (with dequantization) +static void copy_q2_K_row(float * dst, const block_q2_K * src_blocks, int64_t num_elements) { + const int64_t num_blocks = (num_elements + QK_K - 1) / QK_K; + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const int64_t elements_in_block = (block_idx == num_blocks - 1) ? (num_elements - block_idx * QK_K) : QK_K; + + float temp_buffer[QK_K]; + dequantize_q2_K_block(&src_blocks[block_idx], temp_buffer); + + for (int64_t i = 0; i < elements_in_block; i++) { + dst[block_idx * QK_K + i] = temp_buffer[i]; + } + } +} + +// Copy a row of Q3_K data to F32 destination (with dequantization) +static void copy_q3_K_row(float * dst, const block_q3_K * src_blocks, int64_t num_elements) { + const int64_t num_blocks = (num_elements + QK_K - 1) / QK_K; + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const int64_t elements_in_block = (block_idx == num_blocks - 1) ? (num_elements - block_idx * QK_K) : QK_K; + + float temp_buffer[QK_K]; + dequantize_q3_K_block(&src_blocks[block_idx], temp_buffer); + + for (int64_t i = 0; i < elements_in_block; i++) { + dst[block_idx * QK_K + i] = temp_buffer[i]; + } + } +} + +// Copy a row of Q5_K data to F32 destination (with dequantization) +static void copy_q5_K_row(float * dst, const block_q5_K * src_blocks, int64_t num_elements) { + const int64_t num_blocks = (num_elements + QK_K - 1) / QK_K; + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const int64_t elements_in_block = (block_idx == num_blocks - 1) ? (num_elements - block_idx * QK_K) : QK_K; + + float temp_buffer[QK_K]; + dequantize_q5_K_block(&src_blocks[block_idx], temp_buffer); + + for (int64_t i = 0; i < elements_in_block; i++) { + dst[block_idx * QK_K + i] = temp_buffer[i]; + } + } +} + +// Copy a row of Q6_K data to F32 destination (with dequantization) +static void copy_q6_K_row(float * dst, const block_q6_K * src_blocks, int64_t num_elements) { + const int64_t num_blocks = (num_elements + QK_K - 1) / QK_K; + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const int64_t elements_in_block = (block_idx == num_blocks - 1) ? (num_elements - block_idx * QK_K) : QK_K; + + float temp_buffer[QK_K]; + dequantize_q6_K_block(&src_blocks[block_idx], temp_buffer); + + for (int64_t i = 0; i < elements_in_block; i++) { + dst[block_idx * QK_K + i] = temp_buffer[i]; + } + } +} + static void dequantize_q8_0_block_cache_aligned(const block_q8_0 * block, float * dst) { const int8_t * qs_ptr = block->qs; @@ -521,7 +585,8 @@ int entry_point(struct ggml_et_get_rows_params * params, void * env) { } if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_Q8_0 && - src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K) { + src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q6_K && + src0->type != GGML_TYPE_Q2_K && src0->type != GGML_TYPE_Q3_K && src0->type != GGML_TYPE_Q5_K) { return -1; // Unsupported input type } @@ -605,6 +670,31 @@ int entry_point(struct ggml_et_get_rows_params * params, void * env) { const block_q4_K * src_blocks = (const block_q4_K *) src0_data + src_block_offset; float * dst_row = dst_data + dst_offset * ne00; copy_q4_K_row(dst_row, src_blocks, ne00); + } else if (src0->type == GGML_TYPE_Q6_K) { + // Q6_K source: dequantize while copying + const int64_t blocks_per_row = (ne00 + QK_K - 1) / QK_K; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const block_q6_K * src_blocks = (const block_q6_K *) src0_data + src_block_offset; + float * dst_row = dst_data + dst_offset * ne00; + copy_q6_K_row(dst_row, src_blocks, ne00); + } else if (src0->type == GGML_TYPE_Q2_K) { + const int64_t blocks_per_row = (ne00 + QK_K - 1) / QK_K; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const block_q2_K * src_blocks = (const block_q2_K *) src0_data + src_block_offset; + float * dst_row = dst_data + dst_offset * ne00; + copy_q2_K_row(dst_row, src_blocks, ne00); + } else if (src0->type == GGML_TYPE_Q3_K) { + const int64_t blocks_per_row = (ne00 + QK_K - 1) / QK_K; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const block_q3_K * src_blocks = (const block_q3_K *) src0_data + src_block_offset; + float * dst_row = dst_data + dst_offset * ne00; + copy_q3_K_row(dst_row, src_blocks, ne00); + } else if (src0->type == GGML_TYPE_Q5_K) { + const int64_t blocks_per_row = (ne00 + QK_K - 1) / QK_K; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const block_q5_K * src_blocks = (const block_q5_K *) src0_data + src_block_offset; + float * dst_row = dst_data + dst_offset * ne00; + copy_q5_K_row(dst_row, src_blocks, ne00); } } diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K.c new file mode 100644 index 000000000000..9a8323f3f38f --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K.c @@ -0,0 +1,345 @@ +//****************************************************************************** +// MUL_MAT Kernel (Q2_K weights x F32 activations -> F32) +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +// +// Structure mirrors mul_mat_Q4_K.c. Q2_K also packs 256 elements per +// super-block (16 groups of 16) with per-group int8 scales, so the same +// super-block K-tiling applies; only the per-element dequant differs and is +// delegated to Q2K_DOT() in block_ops.h. +//****************************************************************************** + +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "math_fp.h" +#include "quants.h" +#include "block_ops.h" + +#define STRIDE_M 2048 /* 32 shires x 32 minions x 2 harts */ +#define STRIDE_M_KSPLIT 1024 /* 32 shires x 32 minions (both harts share rows) */ +#define KSPLIT_MIN_K_BLOCKS 32 /* K >= 8192 elements (32 super-blocks) */ +#define KSPLIT_SMALL_ROWS_K_BLOCKS 8 /* K >= 2048 elements for very small M */ +#define KSPLIT_MAX_ROWS 8 /* max rows per minion for K-split */ +#define TILE_KB 32 /* K-tile size in Q2_K super-blocks (8192 elems) */ +#define KSPLIT_GROUP_ROWS 4 + +// Vectorized (8-wide) dot. Scalar kept for A/B. +#define Q2K_DOT(a, b, c) compute_row_dot_q2_K_vec(a, b, c) + +#ifdef ET_UBERKERNEL +static inline size_t tensor_bytes(const struct ggml_tensor* t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} +#endif + +int entry_point(struct ggml_et_binary_params* params, void* env) { + uint64_t hart_id = get_hart_id(); + +#ifdef ET_UBERKERNEL + // Uberkernel coherency: src1 (activations) may be stale in L1/L2; force + // re-read from L3/DRAM. src0 (weights) is read-only, never stale. + evict_region_past_l2(params->src1.data, tensor_bytes(¶ms->src1)); +#endif + + // Matrix dimensions + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + const int64_t ne02 = params->src0.ne[2]; + const int64_t ne03 = params->src0.ne[3]; + const int64_t ne12 = params->src1.ne[2]; + const int64_t ne13 = params->src1.ne[3]; + + // Strides (in bytes) + const size_t nb01 = params->src0.nb[1]; + const size_t nb02 = params->src0.nb[2]; + const size_t nb03 = params->src0.nb[3]; + + const size_t nb11 = params->src1.nb[1]; + const size_t nb12 = params->src1.nb[2]; + const size_t nb13 = params->src1.nb[3]; + + const size_t nbd1 = params->dst.nb[1]; + const size_t nbd2 = params->dst.nb[2]; + const size_t nbd3 = params->dst.nb[3]; + + // Q2_K super-block holds 256 elements + const int64_t K_blocks = K / QK_K; + + // Broadcasting ratios + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // K-split decision + const int64_t minion_id = hart_id >> 1; /* 0..1023 global */ + const int64_t local_minion = (hart_id >> 1) & 0x1F; /* 0..31 within shire */ + const int is_hart1 = hart_id & 1; + const int64_t rows_per_minion = (M + STRIDE_M_KSPLIT - 1) / STRIDE_M_KSPLIT; + const int64_t k_half = K_blocks / 2; + const int use_ksplit_small_rows = (rows_per_minion <= 2) + && (K_blocks >= KSPLIT_SMALL_ROWS_K_BLOCKS); + /* + * K-split when K is large enough to benefit, and either: + * - few rows (<=4): always safe, proven working + * - more rows (5-8): only if each hart's half fits in one tile, + * otherwise L1 thrashing from 2 harts x 8 rows kills performance + * + * Also allow K-split earlier for the low-M regime (<=2 rows/minion). In + * that case the simple row-striped path leaves half the machine idle, so + * using both harts on each row pays off even for moderate K. + */ + const int use_ksplit = ((K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion <= KSPLIT_MAX_ROWS) + && (rows_per_minion <= 4 || k_half <= TILE_KB)) + || use_ksplit_small_rows; + const int use_ksplit_group = !use_ksplit + && (K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion > 4) + && (rows_per_minion <= KSPLIT_MAX_ROWS); + + if (use_ksplit) { + /* Each hart processes half the K dimension */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + + /* One cache-line-aligned L2SCP slot per minion for exchange */ + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = minion_id; m < M; m += STRIDE_M_KSPLIT) { + const block_q2_K* q_row = (const block_q2_K*)(src0_ptr2 + m * nb01); + + float partial = Q2K_DOT( + q_row + k_start, b_col_base + k_start * QK_K, k_len); + + if (is_hart1) { + *l2scp_slot = partial; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + float other = *l2scp_slot; + et_sem_post(ET_BARRIER_MINION); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, partial + other); + } + } + } + } + } + } else if (use_ksplit_group) { + /* + * Grouped K-split for the 5-8 rows/minion regime. + * + * Both harts process the same 4-row group, each on half of K, and + * exchange 4 partial sums once per group instead of once per row. + */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m_base = minion_id; m_base < M; + m_base += STRIDE_M_KSPLIT * KSPLIT_GROUP_ROWS) { + const int64_t m0 = m_base; + const int64_t m1 = m0 + STRIDE_M_KSPLIT; + const int64_t m2 = m1 + STRIDE_M_KSPLIT; + const int64_t m3 = m2 + STRIDE_M_KSPLIT; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = k_len - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + if (tile_len <= 0) { + break; + } + const float* b_tile = b_col_base + (k_start + kb) * QK_K; + const int64_t row_kb = k_start + kb; + + if (m0 < M) { + s0 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m0 * nb01) + row_kb, + b_tile, tile_len); + } + if (m1 < M) { + s1 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m1 * nb01) + row_kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m2 * nb01) + row_kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m3 * nb01) + row_kb, + b_tile, tile_len); + } + } + + if (is_hart1) { + l2scp_slot[0] = s0; + l2scp_slot[1] = s1; + l2scp_slot[2] = s2; + l2scp_slot[3] = s3; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + const float p0 = l2scp_slot[0]; + const float p1 = l2scp_slot[1]; + const float p2 = l2scp_slot[2]; + const float p3 = l2scp_slot[3]; + et_sem_post(ET_BARRIER_MINION); + + float* c_base = (float*)(dst_ptr2 + n * nbd1); + if (m0 < M) atomic_store_f32((volatile float*)(c_base + m0), s0 + p0); + if (m1 < M) atomic_store_f32((volatile float*)(c_base + m1), s1 + p1); + if (m2 < M) atomic_store_f32((volatile float*)(c_base + m2), s2 + p2); + if (m3 < M) atomic_store_f32((volatile float*)(c_base + m3), s3 + p3); + } + } + } + } + } + } else if (K_blocks > TILE_KB) { + /* + * Tile-outer with scalar row groups: process up to 4 rows per + * hart sharing each B tile before advancing to the next tile. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * 4) { + const int64_t m1 = m0 + STRIDE_M; + const int64_t m2 = m0 + STRIDE_M * 2; + const int64_t m3 = m0 + STRIDE_M * 3; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = K_blocks - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + const float* b_tile = b_col_base + kb * QK_K; + + s0 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m0 * nb01) + kb, + b_tile, tile_len); + if (m1 < M) { + s1 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m1 * nb01) + kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m2 * nb01) + kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q2K_DOT( + (const block_q2_K*)(src0_ptr2 + m3 * nb01) + kb, + b_tile, tile_len); + } + } + + float* dst_base = (float*)(dst_ptr2 + n * nbd1); + atomic_store_f32((volatile float*)(dst_base + m0), s0); + if (m1 < M) atomic_store_f32((volatile float*)(dst_base + m1), s1); + if (m2 < M) atomic_store_f32((volatile float*)(dst_base + m2), s2); + if (m3 < M) atomic_store_f32((volatile float*)(dst_base + m3), s3); + } + } + } + } + } else { + /* + * Simple path for small K: one row per hart. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q2_K* q_row = (const block_q2_K*)(src0_ptr2 + m * nb01); + + float sum = Q2K_DOT(q_row, b_col_base, K_blocks); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, sum); + } + } + } + } + } + +#ifdef ET_UBERKERNEL + // Publish dst to L3/DRAM for the next uberkernel op. + FENCE; + evict_region_past_l2(params->dst.data, tensor_bytes(¶ms->dst)); + WAIT_CACHEOPS; + FENCE; +#endif + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c new file mode 100644 index 000000000000..b2cc6438f677 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c @@ -0,0 +1,534 @@ +//****************************************************************************** +// MUL_MAT Kernel +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +//****************************************************************************** + +#include +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" +#include "quants.h" +#include "math_fp.h" + +// Q2_K x F32 -> F32 MUL_MAT on the tensor (matrix) engine, TensorFMA32. +// Identical producer/consumer, tiling and tensor-engine loop to +// mul_mat_Q4_K_matrix_engine.c; only the weight dequant differs (Q2_K uses a +// 6-bit quant with a per-16-element int8 scale and no min term). +// Hart 1: dequantize Q2_K weights to FP32 into double-buffered L2 SCP. +// Hart 0: tensor engine compute (FMA, reduce, store). +// +// Two execution paths (selected at runtime by N % TILE_N): +// * REUSE path (N % TILE_N == 0): dequantize each weight K-window ONCE and +// reuse it across ru_n consecutive N-tiles, so the (producer-bound) +// dequant work is cut by ~ru_n. Partial C is round-tripped through an +// L2-SCP scratch between K-windows (the FMA C accumulator is a single fixed +// register-file tile, so multiple output tiles cannot be resident at once). +// * ORIGINAL path (N % TILE_N != 0): one output tile at a time, no reuse. + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +#define TILE_M 16 +#define TILE_N 16 +#define BLOCK_K 32 // one Q2_K group (32 elements) per panel +#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) + +// --- Reuse knobs ---------------------------------------------------------- +// REUSE_MAX caps the L2-SCP C-scratch footprint; the actual reuse factor is +// chosen at runtime as the largest value that still keeps the whole +// machine busy. KWIN is the dequant-cache depth (K-blocks per window). +#ifndef REUSE_MAX +#define REUSE_MAX 15 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per dequant window (cache depth) +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 + +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) + +// Single dequant panel: BLOCK_K k-lines x TILE_M m (FP32) = 32*64 = 2048 bytes, +// [k][m] order: panel[k*TILE_M + m]. +#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t)sizeof(float)) // 2048 + +// L2 SCP layout per minion. The REUSE path needs the larger footprint, so the +// per-minion stride uses it for both paths (mutually exclusive at runtime). +// [0 .. RU_BUF_BYTES) cache buffer 0 (KWIN panels) +// [RU_BUF_BYTES .. 2*..) cache buffer 1 (KWIN panels) +// [RU_CACHE_BYTES .. +R*1024) REUSE_MAX C-scratch tiles (16 rows*64B each) +// ready_ctr, consumed_ctr sync counters +// The ORIGINAL path reuses [0,2048) and [2048,4096) as its two panels and the +// same ready/consumed counters (which sit above the cache region). +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) + +// Software fp16->fp32 (pure integer). The hardware fcvt.ps.f16 returns wrong +// values after the attention block (shared conversion-unit state), which would +// corrupt the weight scales here; software conversion avoids that instruction +// entirely. Only the super-block d is fp16, so cost is negligible. +static inline float __attribute__((always_inline)) me_sw_fp16(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000) << 16; + uint32_t exp = (h >> 10) & 0x1F; + uint32_t mant = h & 0x3FF; + uint32_t f; + if (exp == 0) { + if (mant == 0) { f = sign; } + else { + exp = 127 - 15 + 1; + while ((mant & 0x400) == 0) { mant <<= 1; exp--; } + mant &= 0x3FF; + f = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1F) { + f = sign | 0x7F800000u | (mant << 13); + } else { + f = sign | ((exp + (127 - 15)) << 23) | (mant << 13); + } + float out; __builtin_memcpy(&out, &f, 4); return out; +} + +// EXPERIMENT: use the hardware fcvt.ps.f16 (fp16_to_fp32, math_fp.h) for the +// super-block scale instead of the software me_sw_fp16 above. Hardware fcvt was +// observed to corrupt Q4_K scales after the attention block; trying it here for +// the other K-quants. If it produces garbage, flip ME_FP16 back to me_sw_fp16. +// #define ME_FP16(h) fp16_to_fp32(h) // hardware fcvt: garbage for Q3/Q5/Q6 (fcvt-after-attention bug) +#define ME_FP16(h) me_sw_fp16(h) + +// Dequantize one 32-element Q2_K GROUP of TILE_M weight rows into the FP32 panel, +// written in TenB [k][m] order: panel[k*TILE_M + m]. +// +// A Q2_K super-block packs 256 elements as two 128-element chunks; each 2-bit +// weight q is affine `w = d*(sc&0xF)*q - dmin*(sc>>4)` with a 4-bit scale/min +// pair per 16 elements. kb_group is the global 32-element group index: +// super-block = kb_group/8, group g = kb_group%8, mapping to chunk c = g/4 and +// bit-shift index jj = g%4 (see dequantize_q2_K_block). +static inline void __attribute__((always_inline)) +dequant_q2_K_panel(float *panel, const char *src0_batch, + int64_t mb, int64_t kb_group, int64_t nb1_0) { + const int64_t sb = kb_group >> 3; + const int g = (int) (kb_group & 7); + const int c = g >> 2; // 128-element chunk (0 or 1) + const int jj = g & 3; // sub-group -> shift 2*jj + const int shift = 2 * jj; + const int qoff = c * 32; + const int scoff = c * 8 + 2 * jj; + + for (int j = 0; j < TILE_M; ++j) { + const block_q2_K *blk = + (const block_q2_K *)(src0_batch + (mb + j) * nb1_0) + sb; + const float d = ME_FP16(blk->d); + const float dm = ME_FP16(blk->dmin); + const uint8_t * qs = blk->qs + qoff; + const uint8_t * scl = blk->scales + scoff; + + for (int l = 0; l < 32; ++l) { + const int half = l >> 4; // 0 -> sub A, 1 -> sub B + const uint8_t sc = scl[half]; + const float dl = d * (float) (sc & 0xF); + const float ml = dm * (float) (sc >> 4); + const int8_t q = (int8_t)((qs[l] >> shift) & 3); + panel[l * TILE_M + j] = dl * (float) q - ml; + } + } +} + +// Spill / seed the FP32 C accumulator (16x16 tile in the vector register file, +// row n -> f2n[cols 0..7], f2n+1[cols 8..15]) to/from a 1 KB L2-SCP scratch. +// scratch layout: row n at byte offset n*64. Always moves all 16 rows; rows +// beyond a partial n_cur carry harmless garbage (never stored / recomputed). +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); +} + +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} + +int entry_point(struct ggml_et_binary_params *params, void *env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + // Dimensions (both harts need these for tile assignment) + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + if ((M % TILE_M) != 0) return 0; + if ((K % QK_K) != 0) return 0; + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t k_steps = K / BLOCK_K; // number of 32-element groups + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE; + const int64_t local_tile_idx = local_minion; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + // L2 SCP pointers for this minion. + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // Calculate ru_n to perfectly minimize hardware waves while avoiding Consumer bottleneck. + // The pipeline is perfectly balanced at r=8. Score = waves * max(8, r). + // We find the r that minimizes Score. + int64_t best_r = 1; + int64_t min_score = INT64_MAX; + int64_t max_search_r = REUSE_MAX; + if (max_search_r > n_tiles) max_search_r = n_tiles; + + for (int64_t r = 1; r <= max_search_r; r++) { + int64_t n_groups = (n_tiles + r - 1) / r; + int64_t base_units = m_tiles * n_groups * batch_count; + int64_t waves = (base_units + MACHINE_SLOTS - 1) / MACHINE_SLOTS; + + int64_t penalty = (r > 8) ? r : 8; + int64_t score = waves * penalty; + + if (score < min_score) { + min_score = score; + best_r = r; + } + } + int64_t ru_n = best_r; + + // Reuse pays only when it groups >=2 N-tiles; otherwise the windowing / + // C round-trip is pure overhead, so use the one-tile-at-a-time path. + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: dequant each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t n_groups = (n_tiles + ru_n - 1) / ru_n; + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + const int64_t n_windows = (k_steps + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + dequant_q2_K_panel(cf + i * (SCP_PANEL_SIZE / 4), + src0_batch, mb, kb0 + i, nb1_0); + } + FENCE; + flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } + + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; // first N-tile + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + const int is_last = (kw == n_windows - 1); + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + if (kw > 0) c_seed(cs); + int first = (kw == 0) ? 1 : 0; + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + for (int64_t i = 0; i < kbn; ++i) { + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = (kb0 + i) * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(cf + i * (SCP_PANEL_SIZE / 4) + half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + } + + if (is_last) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } + + // ===================================================================== + // ORIGINAL path: one output tile at a time (N % TILE_N != 0). No reuse. + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { + (float *) et_shire_l2scp_local(scp_base), + (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), + }; + + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); + + dequant_q2_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_panel[buf], BLOCK_K, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + FENCE; + return 0; + } + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t chunk_id = 0; + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + int first = 1; + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + chunk_id++; + scp_wait(ready_ctr, chunk_id); + + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = kb * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + + scp_signal(consumed_ctr, chunk_id); + } + + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K.c new file mode 100644 index 000000000000..41725f41f50c --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K.c @@ -0,0 +1,345 @@ +//****************************************************************************** +// MUL_MAT Kernel (Q3_K weights x F32 activations -> F32) +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +// +// Structure mirrors mul_mat_Q4_K.c. Q3_K also packs 256 elements per +// super-block (16 groups of 16) with per-group int8 scales, so the same +// super-block K-tiling applies; only the per-element dequant differs and is +// delegated to Q3K_DOT() in block_ops.h. +//****************************************************************************** + +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "math_fp.h" +#include "quants.h" +#include "block_ops.h" + +#define STRIDE_M 2048 /* 32 shires x 32 minions x 2 harts */ +#define STRIDE_M_KSPLIT 1024 /* 32 shires x 32 minions (both harts share rows) */ +#define KSPLIT_MIN_K_BLOCKS 32 /* K >= 8192 elements (32 super-blocks) */ +#define KSPLIT_SMALL_ROWS_K_BLOCKS 8 /* K >= 2048 elements for very small M */ +#define KSPLIT_MAX_ROWS 8 /* max rows per minion for K-split */ +#define TILE_KB 32 /* K-tile size in Q3_K super-blocks (8192 elems) */ +#define KSPLIT_GROUP_ROWS 4 + +// Vectorized (8-wide) dot +#define Q3K_DOT(a, b, c) compute_row_dot_q3_K_vec(a, b, c) + +#ifdef ET_UBERKERNEL +static inline size_t tensor_bytes(const struct ggml_tensor* t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} +#endif + +int entry_point(struct ggml_et_binary_params* params, void* env) { + uint64_t hart_id = get_hart_id(); + +#ifdef ET_UBERKERNEL + // Uberkernel coherency: src1 (activations) may be stale in L1/L2; force + // re-read from L3/DRAM. src0 (weights) is read-only, never stale. + evict_region_past_l2(params->src1.data, tensor_bytes(¶ms->src1)); +#endif + + // Matrix dimensions + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + const int64_t ne02 = params->src0.ne[2]; + const int64_t ne03 = params->src0.ne[3]; + const int64_t ne12 = params->src1.ne[2]; + const int64_t ne13 = params->src1.ne[3]; + + // Strides (in bytes) + const size_t nb01 = params->src0.nb[1]; + const size_t nb02 = params->src0.nb[2]; + const size_t nb03 = params->src0.nb[3]; + + const size_t nb11 = params->src1.nb[1]; + const size_t nb12 = params->src1.nb[2]; + const size_t nb13 = params->src1.nb[3]; + + const size_t nbd1 = params->dst.nb[1]; + const size_t nbd2 = params->dst.nb[2]; + const size_t nbd3 = params->dst.nb[3]; + + // Q3_K super-block holds 256 elements + const int64_t K_blocks = K / QK_K; + + // Broadcasting ratios + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // K-split decision + const int64_t minion_id = hart_id >> 1; /* 0..1023 global */ + const int64_t local_minion = (hart_id >> 1) & 0x1F; /* 0..31 within shire */ + const int is_hart1 = hart_id & 1; + const int64_t rows_per_minion = (M + STRIDE_M_KSPLIT - 1) / STRIDE_M_KSPLIT; + const int64_t k_half = K_blocks / 2; + const int use_ksplit_small_rows = (rows_per_minion <= 2) + && (K_blocks >= KSPLIT_SMALL_ROWS_K_BLOCKS); + /* + * K-split when K is large enough to benefit, and either: + * - few rows (<=4): always safe, proven working + * - more rows (5-8): only if each hart's half fits in one tile, + * otherwise L1 thrashing from 2 harts x 8 rows kills performance + * + * Also allow K-split earlier for the low-M regime (<=2 rows/minion). In + * that case the simple row-striped path leaves half the machine idle, so + * using both harts on each row pays off even for moderate K. + */ + const int use_ksplit = ((K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion <= KSPLIT_MAX_ROWS) + && (rows_per_minion <= 4 || k_half <= TILE_KB)) + || use_ksplit_small_rows; + const int use_ksplit_group = !use_ksplit + && (K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion > 4) + && (rows_per_minion <= KSPLIT_MAX_ROWS); + + if (use_ksplit) { + /* Each hart processes half the K dimension */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + + /* One cache-line-aligned L2SCP slot per minion for exchange */ + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = minion_id; m < M; m += STRIDE_M_KSPLIT) { + const block_q3_K* q_row = (const block_q3_K*)(src0_ptr2 + m * nb01); + + float partial = Q3K_DOT( + q_row + k_start, b_col_base + k_start * QK_K, k_len); + + if (is_hart1) { + *l2scp_slot = partial; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + float other = *l2scp_slot; + et_sem_post(ET_BARRIER_MINION); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, partial + other); + } + } + } + } + } + } else if (use_ksplit_group) { + /* + * Grouped K-split for the 5-8 rows/minion regime. + * + * Both harts process the same 4-row group, each on half of K, and + * exchange 4 partial sums once per group instead of once per row. + */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m_base = minion_id; m_base < M; + m_base += STRIDE_M_KSPLIT * KSPLIT_GROUP_ROWS) { + const int64_t m0 = m_base; + const int64_t m1 = m0 + STRIDE_M_KSPLIT; + const int64_t m2 = m1 + STRIDE_M_KSPLIT; + const int64_t m3 = m2 + STRIDE_M_KSPLIT; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = k_len - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + if (tile_len <= 0) { + break; + } + const float* b_tile = b_col_base + (k_start + kb) * QK_K; + const int64_t row_kb = k_start + kb; + + if (m0 < M) { + s0 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m0 * nb01) + row_kb, + b_tile, tile_len); + } + if (m1 < M) { + s1 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m1 * nb01) + row_kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m2 * nb01) + row_kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m3 * nb01) + row_kb, + b_tile, tile_len); + } + } + + if (is_hart1) { + l2scp_slot[0] = s0; + l2scp_slot[1] = s1; + l2scp_slot[2] = s2; + l2scp_slot[3] = s3; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + const float p0 = l2scp_slot[0]; + const float p1 = l2scp_slot[1]; + const float p2 = l2scp_slot[2]; + const float p3 = l2scp_slot[3]; + et_sem_post(ET_BARRIER_MINION); + + float* c_base = (float*)(dst_ptr2 + n * nbd1); + if (m0 < M) atomic_store_f32((volatile float*)(c_base + m0), s0 + p0); + if (m1 < M) atomic_store_f32((volatile float*)(c_base + m1), s1 + p1); + if (m2 < M) atomic_store_f32((volatile float*)(c_base + m2), s2 + p2); + if (m3 < M) atomic_store_f32((volatile float*)(c_base + m3), s3 + p3); + } + } + } + } + } + } else if (K_blocks > TILE_KB) { + /* + * Tile-outer with scalar row groups: process up to 4 rows per + * hart sharing each B tile before advancing to the next tile. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * 4) { + const int64_t m1 = m0 + STRIDE_M; + const int64_t m2 = m0 + STRIDE_M * 2; + const int64_t m3 = m0 + STRIDE_M * 3; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = K_blocks - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + const float* b_tile = b_col_base + kb * QK_K; + + s0 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m0 * nb01) + kb, + b_tile, tile_len); + if (m1 < M) { + s1 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m1 * nb01) + kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m2 * nb01) + kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q3K_DOT( + (const block_q3_K*)(src0_ptr2 + m3 * nb01) + kb, + b_tile, tile_len); + } + } + + float* dst_base = (float*)(dst_ptr2 + n * nbd1); + atomic_store_f32((volatile float*)(dst_base + m0), s0); + if (m1 < M) atomic_store_f32((volatile float*)(dst_base + m1), s1); + if (m2 < M) atomic_store_f32((volatile float*)(dst_base + m2), s2); + if (m3 < M) atomic_store_f32((volatile float*)(dst_base + m3), s3); + } + } + } + } + } else { + /* + * Simple path for small K: one row per hart. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q3_K* q_row = (const block_q3_K*)(src0_ptr2 + m * nb01); + + float sum = Q3K_DOT(q_row, b_col_base, K_blocks); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, sum); + } + } + } + } + } + +#ifdef ET_UBERKERNEL + // Publish dst to L3/DRAM for the next uberkernel op. + FENCE; + evict_region_past_l2(params->dst.data, tensor_bytes(¶ms->dst)); + WAIT_CACHEOPS; + FENCE; +#endif + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c new file mode 100644 index 000000000000..19bf1b03bb6f --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c @@ -0,0 +1,536 @@ +//****************************************************************************** +// MUL_MAT Kernel +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +//****************************************************************************** + +#include +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" +#include "quants.h" +#include "math_fp.h" + +// Q3_K x F32 -> F32 MUL_MAT on the tensor (matrix) engine, TensorFMA32. +// Identical producer/consumer, tiling and tensor-engine loop to +// mul_mat_Q4_K_matrix_engine.c; only the weight dequant differs (Q3_K uses a +// 6-bit quant with a per-16-element int8 scale and no min term). +// Hart 1: dequantize Q3_K weights to FP32 into double-buffered L2 SCP. +// Hart 0: tensor engine compute (FMA, reduce, store). +// +// Two execution paths (selected at runtime by N % TILE_N): +// * REUSE path (N % TILE_N == 0): dequantize each weight K-window ONCE and +// reuse it across ru_n consecutive N-tiles, so the (producer-bound) +// dequant work is cut by ~ru_n. Partial C is round-tripped through an +// L2-SCP scratch between K-windows (the FMA C accumulator is a single fixed +// register-file tile, so multiple output tiles cannot be resident at once). +// * ORIGINAL path (N % TILE_N != 0): one output tile at a time, no reuse. + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +#define TILE_M 16 +#define TILE_N 16 +#define BLOCK_K 32 // one Q3_K group (32 elements) per panel +#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) + +// --- Reuse knobs ---------------------------------------------------------- +// REUSE_MAX caps the L2-SCP C-scratch footprint; the actual reuse factor is +// chosen at runtime as the largest value that still keeps the whole +// machine busy. KWIN is the dequant-cache depth (K-blocks per window). +#ifndef REUSE_MAX +#define REUSE_MAX 15 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per dequant window (cache depth) +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 + +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) + +// Single dequant panel: BLOCK_K k-lines x TILE_M m (FP32) = 32*64 = 2048 bytes, +// [k][m] order: panel[k*TILE_M + m]. +#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t)sizeof(float)) // 2048 + +// L2 SCP layout per minion. The REUSE path needs the larger footprint, so the +// per-minion stride uses it for both paths (mutually exclusive at runtime). +// [0 .. RU_BUF_BYTES) cache buffer 0 (KWIN panels) +// [RU_BUF_BYTES .. 2*..) cache buffer 1 (KWIN panels) +// [RU_CACHE_BYTES .. +R*1024) REUSE_MAX C-scratch tiles (16 rows*64B each) +// ready_ctr, consumed_ctr sync counters +// The ORIGINAL path reuses [0,2048) and [2048,4096) as its two panels and the +// same ready/consumed counters (which sit above the cache region). +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) + +// Software fp16->fp32 (pure integer). The hardware fcvt.ps.f16 returns wrong +// values after the attention block (shared conversion-unit state), which would +// corrupt the weight scales here; software conversion avoids that instruction +// entirely. Only the super-block d is fp16, so cost is negligible. +static inline float __attribute__((always_inline)) me_sw_fp16(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000) << 16; + uint32_t exp = (h >> 10) & 0x1F; + uint32_t mant = h & 0x3FF; + uint32_t f; + if (exp == 0) { + if (mant == 0) { f = sign; } + else { + exp = 127 - 15 + 1; + while ((mant & 0x400) == 0) { mant <<= 1; exp--; } + mant &= 0x3FF; + f = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1F) { + f = sign | 0x7F800000u | (mant << 13); + } else { + f = sign | ((exp + (127 - 15)) << 23) | (mant << 13); + } + float out; __builtin_memcpy(&out, &f, 4); return out; +} + +// EXPERIMENT: use the hardware fcvt.ps.f16 (fp16_to_fp32, math_fp.h) for the +// super-block scale instead of the software me_sw_fp16 above. Hardware fcvt was +// observed to corrupt Q4_K scales after the attention block; trying it here for +// the other K-quants. If it produces garbage, flip ME_FP16 back to me_sw_fp16. +// #define ME_FP16(h) fp16_to_fp32(h) // hardware fcvt: garbage for Q3/Q5/Q6 (fcvt-after-attention bug) +#define ME_FP16(h) me_sw_fp16(h) + +// Dequantize one 32-element Q3_K GROUP of TILE_M weight rows into the FP32 panel, +// written in TenB [k][m] order: panel[k*TILE_M + m]. +// +// A Q3_K super-block packs 256 elements as two 128-element chunks; each 3-bit +// weight is `w = d*(scale-32)*(q2 - (hbit?0:4))` where q2 is 2 low bits from qs +// and hbit is the high bit from hmask; scales are 16 packed 6-bit values. +// kb_group is the global 32-element group index: super-block = kb_group/8, +// group g = kb_group%8, mapping to chunk c = g/4, bit-shift jj = g%4 and hmask +// bit 1<> 3; + const int g = (int) (kb_group & 7); + const int c = g >> 2; // 128-element chunk (0 or 1) + const int jj = g & 3; // sub-group -> shift 2*jj + const int shift = 2 * jj; + const uint8_t hbit = (uint8_t) (1u << g); + const int qoff = c * 32; + const int scoff = c * 8 + 2 * jj; + + for (int j = 0; j < TILE_M; ++j) { + const block_q3_K *blk = + (const block_q3_K *)(src0_batch + (mb + j) * nb1_0) + sb; + const float d = ME_FP16(blk->d); + const uint8_t * qs = blk->qs + qoff; + const uint8_t * hm = blk->hmask; + int8_t scales[16]; + unpack_q3_K_scales(blk->scales, scales); + + for (int l = 0; l < 32; ++l) { + const int half = l >> 4; + const float dl = d * (float) (scales[scoff + half] - 32); + const int8_t low2 = (int8_t)((qs[l] >> shift) & 3); + const int8_t q = (int8_t)(low2 - ((hm[l] & hbit) ? 0 : 4)); + panel[l * TILE_M + j] = dl * (float) q; + } + } +} + +// Spill / seed the FP32 C accumulator (16x16 tile in the vector register file, +// row n -> f2n[cols 0..7], f2n+1[cols 8..15]) to/from a 1 KB L2-SCP scratch. +// scratch layout: row n at byte offset n*64. Always moves all 16 rows; rows +// beyond a partial n_cur carry harmless garbage (never stored / recomputed). +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); +} + +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} + +int entry_point(struct ggml_et_binary_params *params, void *env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + // Dimensions (both harts need these for tile assignment) + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + if ((M % TILE_M) != 0) return 0; + if ((K % QK_K) != 0) return 0; + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t k_steps = K / BLOCK_K; // number of 32-element groups + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE; + const int64_t local_tile_idx = local_minion; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + // L2 SCP pointers for this minion. + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // Calculate ru_n to perfectly minimize hardware waves while avoiding Consumer bottleneck. + // The pipeline is perfectly balanced at r=8. Score = waves * max(8, r). + // We find the r that minimizes Score. + int64_t best_r = 1; + int64_t min_score = INT64_MAX; + int64_t max_search_r = REUSE_MAX; + if (max_search_r > n_tiles) max_search_r = n_tiles; + + for (int64_t r = 1; r <= max_search_r; r++) { + int64_t n_groups = (n_tiles + r - 1) / r; + int64_t base_units = m_tiles * n_groups * batch_count; + int64_t waves = (base_units + MACHINE_SLOTS - 1) / MACHINE_SLOTS; + + int64_t penalty = (r > 8) ? r : 8; + int64_t score = waves * penalty; + + if (score < min_score) { + min_score = score; + best_r = r; + } + } + int64_t ru_n = best_r; + + // Reuse pays only when it groups >=2 N-tiles; otherwise the windowing / + // C round-trip is pure overhead, so use the one-tile-at-a-time path. + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: dequant each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t n_groups = (n_tiles + ru_n - 1) / ru_n; + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + const int64_t n_windows = (k_steps + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + dequant_q3_K_panel(cf + i * (SCP_PANEL_SIZE / 4), + src0_batch, mb, kb0 + i, nb1_0); + } + FENCE; + flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } + + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; // first N-tile + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + const int is_last = (kw == n_windows - 1); + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + if (kw > 0) c_seed(cs); + int first = (kw == 0) ? 1 : 0; + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + for (int64_t i = 0; i < kbn; ++i) { + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = (kb0 + i) * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(cf + i * (SCP_PANEL_SIZE / 4) + half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + } + + if (is_last) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } + + // ===================================================================== + // ORIGINAL path: one output tile at a time (N % TILE_N != 0). No reuse. + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { + (float *) et_shire_l2scp_local(scp_base), + (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), + }; + + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); + + dequant_q3_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_panel[buf], BLOCK_K, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + FENCE; + return 0; + } + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t chunk_id = 0; + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + int first = 1; + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + chunk_id++; + scp_wait(ready_ctr, chunk_id); + + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = kb * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + + scp_signal(consumed_ctr, chunk_id); + } + + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c index 28a10303235a..7a75ea95e9c8 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c @@ -1,56 +1,68 @@ +#include +#include #include "ggml_tensor.h" -#include "math_fp.h" #include "platform.h" -#include "quants.h" #include "tensor.h" - -#include -#include +#include "quants.h" +#include "math_fp.h" // Q4_0 x F32 -> F32 MUL_MAT on the tensor (matrix) engine, TensorFMA32. // Hart 1: dequantize Q4_0 weights to FP32 into double-buffered L2 SCP. // Hart 0: tensor engine compute (FMA, reduce, store). +// +// Two execution paths (selected at runtime by N % TILE_N): +// * REUSE path (N % TILE_N == 0): dequantize each weight K-window ONCE and +// reuse it across ru_n consecutive N-tiles, so the (producer-bound) +// dequant work is cut by ~ru_n. Partial C is round-tripped through an +// L2-SCP scratch between K-windows (the FMA C accumulator is a single fixed +// register-file tile, so multiple output tiles cannot be resident at once). +// * ORIGINAL path (N % TILE_N != 0): one output tile at a time, no reuse. #define NUM_COMPUTE_SHIRES 32 #define MINIONS_PER_SHIRE 32 #define TILE_M 16 #define TILE_N 16 -#define BLOCK_K QK4_0 // 32 elements per Q4_0 block -#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) +#define BLOCK_K QK4_0 // 32 elements per Q4_0 block +#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) + +// --- Reuse knobs ---------------------------------------------------------- +// REUSE_MAX caps the L2-SCP C-scratch footprint; the actual reuse factor is +// chosen at runtime (see ru_n) as the largest value that still keeps the whole +// machine busy. KWIN is the dequant-cache depth (K-blocks per window). +#ifndef REUSE_MAX +#define REUSE_MAX 15 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per dequant window (cache depth) +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 #define CACHEOP_MAX 0 #define REP_RATE 0 -#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) -#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) +#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) -// L2 SCP layout per minion (double-buffered dequant panel + sync counters). -// panel = BLOCK_K k-lines x TILE_M m (FP32) = 32 * 64 = 2048 bytes, in TenB +// Single dequant panel: BLOCK_K k-lines x TILE_M m (FP32) = 32*64 = 2048 bytes, // [k][m] order: panel[k*TILE_M + m]. -#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t) sizeof(float)) // 2048 -#define SCP_READY_OFF (2 * SCP_PANEL_SIZE) // 4096 -#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 4160 -#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 4224 - -// Signal a counter value to the other hart via L2 SCP. -static inline void __attribute__((always_inline)) scp_signal(volatile uint32_t * flag, uint32_t value) { - *flag = value; - FENCE; - evict_to_l2((const void *) flag, 1, 64); - WAIT_CACHEOPS; -} - -// Wait for a counter in L2 SCP to reach the expected value. -static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * flag, uint32_t expected) { - while (1) { - evict_to_l2((const void *) flag, 1, 64); - WAIT_CACHEOPS; - if (*flag >= expected) { - return; - } - } -} +#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t)sizeof(float)) // 2048 + +// L2 SCP layout per minion. The REUSE path needs the larger footprint, so the +// per-minion stride uses it for both paths (mutually exclusive at runtime). +// [0 .. RU_BUF_BYTES) cache buffer 0 (KWIN panels) +// [RU_BUF_BYTES .. 2*..) cache buffer 1 (KWIN panels) +// [RU_CACHE_BYTES .. +R*1024) REUSE_MAX C-scratch tiles (16 rows*64B each) +// ready_ctr, consumed_ctr sync counters +// The ORIGINAL path reuses [0,2048) and [2048,4096) as its two panels and the +// same ready/consumed counters (which sit above the cache region). +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // Dequantize one 32-element Q4_0 block of TILE_M weight rows into the FP32 // panel, written directly in TenB [k][m] order: panel[k*TILE_M + m]. @@ -62,97 +74,129 @@ static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * f // the low/high nibbles to FP32 (nibble-8), scale by the block's fp16 d, and // fscw.ps-scatter the 8 values down 8 panel lines (stride 64B) at column m. // 4 groups of 8 cover the 32 k-values (low 0..15, high 16..31). -static inline void __attribute__((always_inline)) dequant_q4_0_panel(float * panel, - const char * src0_batch, - int64_t mb, - int64_t kb_block, - int64_t nb1_0) { +static inline void __attribute__((always_inline)) +dequant_q4_0_panel(float *panel, const char *src0_batch, + int64_t mb, int64_t kb_block, int64_t nb1_0) { static const int32_t __attribute__((aligned(32))) scatter_idx[8] = { - 0, 64, 128, 192, 256, 320, 384, 448 // byte offsets: 8 lines apart + 0, 64, 128, 192, 256, 320, 384, 448 // byte offsets: 8 lines apart }; static const int32_t __attribute__((aligned(32))) gather_idx[8] = { - 0, 1, 2, 3, 4, 5, 6, 7 // 8 consecutive bytes + 0, 1, 2, 3, 4, 5, 6, 7 // 8 consecutive bytes }; unsigned long old_mask; __asm__ volatile( "mova.x.m %[ms] \n\t" - "mov.m.x m0, x0, 0xFF \n\t" // all 8 lanes active - "flw.ps f1, (%[sidx]) \n\t" // f1 = scatter offsets - "flw.ps f2, (%[gidx]) \n\t" // f2 = gather offsets + "mov.m.x m0, x0, 0xFF \n\t" // all 8 lanes active + "flw.ps f1, (%[sidx]) \n\t" // f1 = scatter offsets + "flw.ps f2, (%[gidx]) \n\t" // f2 = gather offsets : [ms] "=&r"(old_mask) : [sidx] "r"(scatter_idx), [gidx] "r"(gather_idx) - : "f1", "f2"); + : "f1", "f2" + ); - char * pbase = (char *) panel; + char *pbase = (char *) panel; for (int j = 0; j < TILE_M; ++j) { - const block_q4_0 * blk = (const block_q4_0 *) (src0_batch + (mb + j) * nb1_0) + kb_block; - uint32_t scale_raw = (uint32_t) blk->d; - const uint8_t * qs = blk->qs; - char * col = pbase + j * 4; // column m=j of the panel + const block_q4_0 *blk = + (const block_q4_0 *)(src0_batch + (mb + j) * nb1_0) + kb_block; + uint32_t scale_raw = (uint32_t) blk->d; + const uint8_t *qs = blk->qs; + char *col = pbase + j * 4; // column m=j of the panel __asm__ volatile( - "fbcx.ps f3, %[sb] \n\t" // broadcast fp16 scale bits - "fcvt.ps.f16 f3, f3 \n\t" // -> d in all 8 lanes (fp32) + "fbcx.ps f3, %[sb] \n\t" // broadcast fp16 scale bits + "fcvt.ps.f16 f3, f3 \n\t" // -> d in all 8 lanes (fp32) - "fgb.ps f4, f2(%[qs0]) \n\t" // gather qs[0..7] - "fandi.pi f5, f4, 15 \n\t" // low nibble + "fgb.ps f4, f2(%[qs0]) \n\t" // gather qs[0..7] + "fandi.pi f5, f4, 15 \n\t" // low nibble "faddi.pi f5, f5, -8 \n\t" "fcvt.ps.pw f5, f5, rne \n\t" "fmul.ps f5, f5, f3 \n\t" - "fscw.ps f5, f1(%[c0]) \n\t" // k=0..7 -> lines 0..7 - "fsrli.pi f6, f4, 4 \n\t" // high nibble + "fscw.ps f5, f1(%[c0]) \n\t" // k=0..7 -> lines 0..7 + "fsrli.pi f6, f4, 4 \n\t" // high nibble "fandi.pi f6, f6, 15 \n\t" "faddi.pi f6, f6, -8 \n\t" "fcvt.ps.pw f6, f6, rne \n\t" "fmul.ps f6, f6, f3 \n\t" - "fscw.ps f6, f1(%[c16]) \n\t" // k=16..23 -> lines 16..23 + "fscw.ps f6, f1(%[c16]) \n\t" // k=16..23 -> lines 16..23 - "fgb.ps f4, f2(%[qs8]) \n\t" // gather qs[8..15] + "fgb.ps f4, f2(%[qs8]) \n\t" // gather qs[8..15] "fandi.pi f5, f4, 15 \n\t" "faddi.pi f5, f5, -8 \n\t" "fcvt.ps.pw f5, f5, rne \n\t" "fmul.ps f5, f5, f3 \n\t" - "fscw.ps f5, f1(%[c8]) \n\t" // k=8..15 -> lines 8..15 + "fscw.ps f5, f1(%[c8]) \n\t" // k=8..15 -> lines 8..15 "fsrli.pi f6, f4, 4 \n\t" "fandi.pi f6, f6, 15 \n\t" "faddi.pi f6, f6, -8 \n\t" "fcvt.ps.pw f6, f6, rne \n\t" "fmul.ps f6, f6, f3 \n\t" - "fscw.ps f6, f1(%[c24]) \n\t" // k=24..31 -> lines 24..31 + "fscw.ps f6, f1(%[c24]) \n\t" // k=24..31 -> lines 24..31 : - : [sb] "r"(scale_raw), [qs0] "r"(qs), [qs8] "r"(qs + 8), [c0] "r"(col), [c8] "r"(col + 8 * 64), + : [sb] "r"(scale_raw), + [qs0] "r"(qs), [qs8] "r"(qs + 8), + [c0] "r"(col), [c8] "r"(col + 8 * 64), [c16] "r"(col + 16 * 64), [c24] "r"(col + 24 * 64) - : "f3", "f4", "f5", "f6", "memory"); + : "f3", "f4", "f5", "f6", "memory" + ); } - __asm__ volatile("mova.m.x %0" ::"r"(old_mask)); + __asm__ volatile("mova.m.x %0" :: "r"(old_mask)); +} + +// Spill / seed the FP32 C accumulator (16x16 tile in the vector register file, +// row n -> f2n[cols 0..7], f2n+1[cols 8..15]) to/from a 1 KB L2-SCP scratch. +// scratch layout: row n at byte offset n*64. Always moves all 16 rows; rows +// beyond a partial n_cur carry harmless garbage (never stored / recomputed). +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); } -int entry_point(struct ggml_et_binary_params * params, void * env) { +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} + +int entry_point(struct ggml_et_binary_params *params, void *env) { (void) env; uint64_t hart_id = get_hart_id(); uint64_t shire_id = get_shire_id(); - if (shire_id >= NUM_COMPUTE_SHIRES) { - return 0; - } + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; - const int is_hart1 = hart_id & 1; - uint64_t local_minion = (hart_id >> 1) & 0x1F; + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; // Dimensions (both harts need these for tile assignment) const int64_t K = params->src0.ne[0]; const int64_t M = params->src0.ne[1]; const int64_t N = params->src1.ne[1]; - if ((M % TILE_M) != 0) { - return 0; - } - if ((K % BLOCK_K) != 0) { - return 0; - } + if ((M % TILE_M) != 0) return 0; + if ((K % BLOCK_K) != 0) return 0; const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; @@ -166,73 +210,241 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const int64_t nb1_d = params->dst.nb[1]; const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; - const char * src0_base = (const char *) params->src0.data; - const char * src1_base = (const char *) params->src1.data; - char * dst_base = (char *) params->dst.data; + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; - const int64_t m_tiles = M / TILE_M; - const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; const int64_t batch_count = ne2_1 * ne3_1; - const int64_t base_tiles = m_tiles * n_tiles * batch_count; const int64_t r2 = ne2_1 / ne2_0; const int64_t r3 = ne3_1 / ne3_0; - const int64_t k_steps = K / BLOCK_K; // number of Q4_0 blocks + const int64_t k_steps = K / BLOCK_K; // number of Q4_0 blocks - // Force a single K-split. - const int64_t k_splits = 1; - - const int64_t tiles_per_shire = MINIONS_PER_SHIRE / k_splits; - const int64_t k_split = local_minion % k_splits; - const int64_t local_tile_idx = local_minion / k_splits; + const int64_t tiles_per_shire = MINIONS_PER_SHIRE; + const int64_t local_tile_idx = local_minion; const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + // L2 SCP pointers for this minion. + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // Calculate ru_n to perfectly minimize hardware waves while avoiding Consumer bottleneck. + // The pipeline is perfectly balanced at r=8. Score = waves * max(8, r). + // We find the r that minimizes Score. + int64_t best_r = 1; + int64_t min_score = INT64_MAX; + int64_t max_search_r = REUSE_MAX; + if (max_search_r > n_tiles) max_search_r = n_tiles; + + for (int64_t r = 1; r <= max_search_r; r++) { + int64_t n_groups = (n_tiles + r - 1) / r; + int64_t base_units = m_tiles * n_groups * batch_count; + int64_t waves = (base_units + MACHINE_SLOTS - 1) / MACHINE_SLOTS; + + int64_t penalty = (r > 8) ? r : 8; + int64_t score = waves * penalty; + + if (score < min_score) { + min_score = score; + best_r = r; + } + } + int64_t ru_n = best_r; + + // Reuse pays only when it groups >=2 N-tiles; otherwise the windowing / + // C round-trip is pure overhead, so use the one-tile-at-a-time path. + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: dequant each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t n_groups = (n_tiles + ru_n - 1) / ru_n; + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + const int64_t n_windows = (k_steps + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + dequant_q4_0_panel(cf + i * (SCP_PANEL_SIZE / 4), + src0_batch, mb, kb0 + i, nb1_0); + } + FENCE; + flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } - const int64_t k_steps_per_split = k_steps / k_splits; - const int64_t kb_start = k_split * k_steps_per_split; // first block - const int64_t kb_end = kb_start + k_steps_per_split; // one past last + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; // first N-tile + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + const int is_last = (kw == n_windows - 1); + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + if (kw > 0) c_seed(cs); + int first = (kw == 0) ? 1 : 0; + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + for (int64_t i = 0; i < kbn; ++i) { + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = (kb0 + i) * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(cf + i * (SCP_PANEL_SIZE / 4) + half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + } + + if (is_last) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } - // L2 SCP pointers for this minion's double-buffered panels + sync. - uint64_t scp_base = local_minion * SCP_PER_MINION; - float * scp_panel[2] = { + // ===================================================================== + // ORIGINAL path: one output tile at a time (N % TILE_N != 0). No reuse. + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { (float *) et_shire_l2scp_local(scp_base), (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), }; - volatile uint32_t * ready_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); - volatile uint32_t * consumed_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); - // ================================================================ - // Hart 1: Q4_0 weight dequant producer - // ================================================================ if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); - uint32_t chunk_id = 0; - for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; - tile += tiles_stride) { + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { const int64_t tiles_per_batch = m_tiles * n_tiles; const int64_t batch_idx = tile / tiles_per_batch; const int64_t tile_in_batch = tile % tiles_per_batch; - - const int64_t mb_idx = tile_in_batch % m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; const int64_t i3 = batch_idx / ne2_1; const int64_t i2 = batch_idx % ne2_1; const int64_t i2_0 = i2 / r2; const int64_t i3_0 = i3 / r3; - const char * src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; - const int64_t mb = mb_idx * TILE_M; + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; - for (int64_t kb = kb_start; kb < kb_end; ++kb) { + for (int64_t kb = 0; kb < k_steps; ++kb) { int buf = chunk_id & 1; - - // Back-pressure: wait for hart 0 to finish with this buffer. - if (chunk_id >= 2) { - scp_wait(consumed_ctr, chunk_id - 1); - } + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); dequant_q4_0_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); @@ -244,123 +456,79 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { scp_signal(ready_ctr, chunk_id); } } - FENCE; return 0; } - // ================================================================ - // Hart 0: tensor engine compute - // ================================================================ - uint64_t my_minion_id = get_minion_id(); - const uint64_t group_base_global = my_minion_id - k_split; - setup_cache_scp(); #if CACHEOP_MAX > 0 || REP_RATE > 0 ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; - - evict_to_l2((const void *) ready_ctr, 1, 64); - WAIT_CACHEOPS; - evict_to_l2((const void *) consumed_ctr, 1, 64); - WAIT_CACHEOPS; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; uint32_t chunk_id = 0; - - for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; - tile += tiles_stride) { + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { const int64_t tiles_per_batch = m_tiles * n_tiles; const int64_t batch_idx = tile / tiles_per_batch; const int64_t tile_in_batch = tile % tiles_per_batch; - - const int64_t nb_idx = tile_in_batch / m_tiles; - const int64_t mb_idx = tile_in_batch % m_tiles; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; const int64_t i3 = batch_idx / ne2_1; const int64_t i2 = batch_idx % ne2_1; - const char * src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; - char * dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; - const int64_t mb = mb_idx * TILE_M; - const int64_t nb = nb_idx * TILE_N; + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); - - // Partial-N tiles run TensorFMA32 with a_num_rows = n_cur-1. - // Errata Type D workaround for n_cur == 4 (AROWS==3): pad A to AROWS==4. const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); if (n_cur == 4) { - // Zero the padded 5th A row (line A_L1_START+4) once; the per-pass A - // load only writes lines A_L1_START..+3, so this persists. - static const float __attribute__((aligned(64))) zero_line[16] = { 0 }; - tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, (uint64_t) zero_line, 0, - 0, // 1 line - 64, 0); + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); tensor_wait(TENSOR_LOAD_WAIT_0); } - int first = 1; // first_pass=1 only for the very first FMA of the tile - - for (int64_t kb = kb_start; kb < kb_end; ++kb) { + int first = 1; + for (int64_t kb = 0; kb < k_steps; ++kb) { int buf = chunk_id & 1; - - // Wait for hart 1 to finish dequantizing this block. chunk_id++; scp_wait(ready_ctr, chunk_id); - // Two FMA passes over the 32-wide block (16 K-cols each). for (int half = 0; half < 2; ++half) { const int64_t k_elem = kb * BLOCK_K + half * FMA_K; - - // Load A (activations) for this 16-K sub-tile, PLAIN. - tensor_load(false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, - (uint64_t) (src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), 0, n_cur - 1, - (uint64_t) nb1_1, 0); - - // Load B (dequantized weights) half from L2 SCP panel, PLAIN. - tensor_load(false, false, B_L1_START, TENSOR_LOAD_PLAIN, 0, - (uint64_t) (scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), 0, FMA_K - 1, 64, 1); - + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); tensor_wait(TENSOR_LOAD_WAIT_0); - tensor_wait(TENSOR_LOAD_WAIT_1); - tensor_fma(false, - 3, // b_num_col: (16/4)-1 - arows_fma, // a_num_rows (n_cur-1, or 4 for the n_cur==4 errata pad) - FMA_K - 1, // a_num_cols - 0, false, false, false, false, B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_load_setup_b( + false, + (uint64_t)(scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); tensor_wait(TENSOR_FMA_WAIT); first = 0; } - // Signal that this buffer is free for hart 1 to reuse. scp_signal(consumed_ctr, chunk_id); } - // K-split ring reduce. - if (k_splits > 1) { - const uint64_t num_regs = (uint64_t) n_cur * 2; - - if (k_split > 0) { - tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, group_base_global + k_split - 1); - tensor_wait(TENSOR_REDUCE_WAIT); - } - - if (k_split < k_splits - 1) { - tensor_reduce_send(0, num_regs, group_base_global + k_split + 1); - tensor_wait(TENSOR_REDUCE_WAIT); - } - } - - // Store FP32 result tile (only the last k-split owns the final sum). - if (k_split == k_splits - 1) { - tensor_store(0, 0, 3, n_cur - 1, (uint64_t) (dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), 0, - (uint64_t) nb1_d); - tensor_wait(TENSOR_STORE_WAIT); - } + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); } FENCE; diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K.c new file mode 100644 index 000000000000..fd29144bc245 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K.c @@ -0,0 +1,348 @@ +//****************************************************************************** +// MUL_MAT Kernel (Q4_K weights x F32 activations -> F32) +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +// +// Structure mirrors mul_mat_Q4_0.c. The key difference is the block size: +// Q4_K packs 256 elements per super-block (8 groups of 32) with per-group +// scales/mins, versus 32 elements per block for Q4_0. The actual dequant + +// dot work is delegated to Q4K_DOT() in block_ops.h. The K-tiling +// and K-split thresholds below are expressed in super-blocks but chosen so the +// element-level behaviour matches the Q4_0 kernel (one Q4_K super-block == 8 +// Q4_0 blocks, so the block thresholds are divided by 8). +//****************************************************************************** + +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "math_fp.h" +#include "quants.h" +#include "block_ops.h" + +#define STRIDE_M 2048 /* 32 shires x 32 minions x 2 harts */ +#define STRIDE_M_KSPLIT 1024 /* 32 shires x 32 minions (both harts share rows) */ +#define KSPLIT_MIN_K_BLOCKS 32 /* K >= 8192 elements (32 super-blocks) */ +#define KSPLIT_SMALL_ROWS_K_BLOCKS 8 /* K >= 2048 elements for very small M */ +#define KSPLIT_MAX_ROWS 8 /* max rows per minion for K-split */ +#define TILE_KB 32 /* K-tile size in Q4_K super-blocks (8192 elems, 32KB B data) */ +#define KSPLIT_GROUP_ROWS 4 + +// Vectorized (8-wide) dot +#define Q4K_DOT(a, b, c) compute_row_dot_q4_K_vec(a, b, c) + +#ifdef ET_UBERKERNEL +static inline size_t tensor_bytes(const struct ggml_tensor* t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} +#endif + +int entry_point(struct ggml_et_binary_params* params, void* env) { + uint64_t hart_id = get_hart_id(); + +#ifdef ET_UBERKERNEL + // Uberkernel coherency: src1 (activations) may be stale in L1/L2 from a + // prior op's write; force re-read from L3/DRAM. src0 (weights) is read-only. + evict_region_past_l2(params->src1.data, tensor_bytes(¶ms->src1)); +#endif + + // Matrix dimensions + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + const int64_t ne02 = params->src0.ne[2]; + const int64_t ne03 = params->src0.ne[3]; + const int64_t ne12 = params->src1.ne[2]; + const int64_t ne13 = params->src1.ne[3]; + + // Strides (in bytes) + const size_t nb01 = params->src0.nb[1]; + const size_t nb02 = params->src0.nb[2]; + const size_t nb03 = params->src0.nb[3]; + + const size_t nb11 = params->src1.nb[1]; + const size_t nb12 = params->src1.nb[2]; + const size_t nb13 = params->src1.nb[3]; + + const size_t nbd1 = params->dst.nb[1]; + const size_t nbd2 = params->dst.nb[2]; + const size_t nbd3 = params->dst.nb[3]; + + // Q4_K super-block holds 256 elements + const int64_t K_blocks = K / QK_K; + + // Broadcasting ratios + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // K-split decision + const int64_t minion_id = hart_id >> 1; /* 0..1023 global */ + const int64_t local_minion = (hart_id >> 1) & 0x1F; /* 0..31 within shire */ + const int is_hart1 = hart_id & 1; + const int64_t rows_per_minion = (M + STRIDE_M_KSPLIT - 1) / STRIDE_M_KSPLIT; + const int64_t k_half = K_blocks / 2; + const int use_ksplit_small_rows = (rows_per_minion <= 2) + && (K_blocks >= KSPLIT_SMALL_ROWS_K_BLOCKS); + /* + * K-split when K is large enough to benefit, and either: + * - few rows (≤4): always safe, proven working + * - more rows (5-8): only if each hart's half fits in one tile, + * otherwise L1 thrashing from 2 harts × 8 rows kills performance + * + * Also allow K-split earlier for the low-M regime (≤2 rows/minion). In + * that case the simple row-striped path leaves half the machine idle, so + * using both harts on each row pays off even for moderate K. + */ + const int use_ksplit = ((K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion <= KSPLIT_MAX_ROWS) + && (rows_per_minion <= 4 || k_half <= TILE_KB)) + || use_ksplit_small_rows; + const int use_ksplit_group = !use_ksplit + && (K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion > 4) + && (rows_per_minion <= KSPLIT_MAX_ROWS); + + if (use_ksplit) { + /* Each hart processes half the K dimension */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + + /* One cache-line-aligned L2SCP slot per minion for exchange */ + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = minion_id; m < M; m += STRIDE_M_KSPLIT) { + const block_q4_K* q_row = (const block_q4_K*)(src0_ptr2 + m * nb01); + + float partial = Q4K_DOT( + q_row + k_start, b_col_base + k_start * QK_K, k_len); + + if (is_hart1) { + *l2scp_slot = partial; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + float other = *l2scp_slot; + et_sem_post(ET_BARRIER_MINION); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, partial + other); + } + } + } + } + } + } else if (use_ksplit_group) { + /* + * Grouped K-split for the 5-8 rows/minion regime. + * + * Both harts process the same 4-row group, each on half of K, and + * exchange 4 partial sums once per group instead of once per row. + */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m_base = minion_id; m_base < M; + m_base += STRIDE_M_KSPLIT * KSPLIT_GROUP_ROWS) { + const int64_t m0 = m_base; + const int64_t m1 = m0 + STRIDE_M_KSPLIT; + const int64_t m2 = m1 + STRIDE_M_KSPLIT; + const int64_t m3 = m2 + STRIDE_M_KSPLIT; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = k_len - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + if (tile_len <= 0) { + break; + } + const float* b_tile = b_col_base + (k_start + kb) * QK_K; + const int64_t row_kb = k_start + kb; + + if (m0 < M) { + s0 += Q4K_DOT( + (const block_q4_K*)(src0_ptr2 + m0 * nb01) + row_kb, + b_tile, tile_len); + } + if (m1 < M) { + s1 += Q4K_DOT( + (const block_q4_K*)(src0_ptr2 + m1 * nb01) + row_kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q4K_DOT( + (const block_q4_K*)(src0_ptr2 + m2 * nb01) + row_kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q4K_DOT( + (const block_q4_K*)(src0_ptr2 + m3 * nb01) + row_kb, + b_tile, tile_len); + } + } + + if (is_hart1) { + l2scp_slot[0] = s0; + l2scp_slot[1] = s1; + l2scp_slot[2] = s2; + l2scp_slot[3] = s3; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + const float p0 = l2scp_slot[0]; + const float p1 = l2scp_slot[1]; + const float p2 = l2scp_slot[2]; + const float p3 = l2scp_slot[3]; + et_sem_post(ET_BARRIER_MINION); + + float* c_base = (float*)(dst_ptr2 + n * nbd1); + if (m0 < M) atomic_store_f32((volatile float*)(c_base + m0), s0 + p0); + if (m1 < M) atomic_store_f32((volatile float*)(c_base + m1), s1 + p1); + if (m2 < M) atomic_store_f32((volatile float*)(c_base + m2), s2 + p2); + if (m3 < M) atomic_store_f32((volatile float*)(c_base + m3), s3 + p3); + } + } + } + } + } + } else if (K_blocks > TILE_KB) { + /* + * Tile-outer with scalar row groups: process up to 4 rows per + * hart sharing each B tile before advancing to the next tile. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * 4) { + const int64_t m1 = m0 + STRIDE_M; + const int64_t m2 = m0 + STRIDE_M * 2; + const int64_t m3 = m0 + STRIDE_M * 3; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = K_blocks - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + const float* b_tile = b_col_base + kb * QK_K; + + s0 += Q4K_DOT( + (const block_q4_K*)(src0_ptr2 + m0 * nb01) + kb, + b_tile, tile_len); + if (m1 < M) { + s1 += Q4K_DOT( + (const block_q4_K*)(src0_ptr2 + m1 * nb01) + kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q4K_DOT( + (const block_q4_K*)(src0_ptr2 + m2 * nb01) + kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q4K_DOT( + (const block_q4_K*)(src0_ptr2 + m3 * nb01) + kb, + b_tile, tile_len); + } + } + + float* dst_base = (float*)(dst_ptr2 + n * nbd1); + atomic_store_f32((volatile float*)(dst_base + m0), s0); + if (m1 < M) atomic_store_f32((volatile float*)(dst_base + m1), s1); + if (m2 < M) atomic_store_f32((volatile float*)(dst_base + m2), s2); + if (m3 < M) atomic_store_f32((volatile float*)(dst_base + m3), s3); + } + } + } + } + } else { + /* + * Simple path for small K: one row per hart. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q4_K* q_row = (const block_q4_K*)(src0_ptr2 + m * nb01); + + float sum = Q4K_DOT(q_row, b_col_base, K_blocks); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, sum); + } + } + } + } + } + +#ifdef ET_UBERKERNEL + // Publish dst to L3/DRAM so the next uberkernel op reads fresh data. + FENCE; + evict_region_past_l2(params->dst.data, tensor_bytes(¶ms->dst)); + WAIT_CACHEOPS; + FENCE; +#endif + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c new file mode 100644 index 000000000000..f8f5eb4d3350 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c @@ -0,0 +1,594 @@ +//****************************************************************************** +// MUL_MAT Kernel +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +//****************************************************************************** + +#include +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" +#include "quants.h" +#include "math_fp.h" + +// Q4_K x F32 -> F32 MUL_MAT on the tensor (matrix) engine, TensorFMA32. +// identical to mul_mat_Q4_0_matrix_engine.c, identical producer/consumer, tiling and +// tensor-engine loop; only the weight dequant differs (Q4_K affine per-group). +// Hart 1: dequantize Q4_K weights to FP32 into double-buffered L2 SCP. +// Hart 0: tensor engine compute (FMA, reduce, store). +// +// Two execution paths (selected at runtime by N % TILE_N): +// * REUSE path (N % TILE_N == 0): dequantize each weight K-window ONCE and +// reuse it across ru_n consecutive N-tiles, so the (producer-bound) +// dequant work is cut by ~ru_n. Partial C is round-tripped through an +// L2-SCP scratch between K-windows (the FMA C accumulator is a single fixed +// register-file tile, so multiple output tiles cannot be resident at once). +// * ORIGINAL path (N % TILE_N != 0): one output tile at a time, no reuse. + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +#define TILE_M 16 +#define TILE_N 16 +#define BLOCK_K 32 // one Q4_K group (32 elements) per panel +#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) + +// --- Reuse knobs ---------------------------------------------------------- +// REUSE_MAX caps the L2-SCP C-scratch footprint; the actual reuse factor is +// chosen at runtime as the largest value that still keeps the whole +// machine busy. KWIN is the dequant-cache depth (K-blocks per window). +#ifndef REUSE_MAX +#define REUSE_MAX 15 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per dequant window (cache depth) +#endif +// BAL is the producer/consumer balance point: reuse factors r<=BAL are treated +// as "free" (dequant is the bottleneck), r>BAL is penalized because the +// consumer (+ C round-trip) becomes the bottleneck. TIE_PREFER_LARGE selects +// the LARGEST r among equal scores (max reuse / least dequant) instead of the +// smallest. +#ifndef BAL +#define BAL 8 +#endif +#ifndef TIE_PREFER_LARGE +#define TIE_PREFER_LARGE 0 +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 + +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) + +// Single dequant panel: BLOCK_K k-lines x TILE_M m (FP32) = 32*64 = 2048 bytes, +// [k][m] order: panel[k*TILE_M + m]. +#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t)sizeof(float)) // 2048 + +// L2 SCP layout per minion. The REUSE path needs the larger footprint, so the +// per-minion stride uses it for both paths (mutually exclusive at runtime). +// [0 .. RU_BUF_BYTES) cache buffer 0 (KWIN panels) +// [RU_BUF_BYTES .. 2*..) cache buffer 1 (KWIN panels) +// [RU_CACHE_BYTES .. +R*1024) REUSE_MAX C-scratch tiles (16 rows*64B each) +// ready_ctr, consumed_ctr sync counters +// The ORIGINAL path reuses [0,2048) and [2048,4096) as its two panels and the +// same ready/consumed counters (which sit above the cache region). +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) + +// Software fp16->fp32 (pure integer). The hardware fcvt.ps.f16 returns wrong +// values after the attention block (shared conversion-unit state), which would +// corrupt the Q4_K weight scales here; software conversion avoids that +// instruction entirely. Only 2 conversions per super-block, so cost is negligible. +// Since this is only observed for Q4_K and only after the attention block, we need +// to investigate it further. +static inline float __attribute__((always_inline)) me_sw_fp16(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000) << 16; + uint32_t exp = (h >> 10) & 0x1F; + uint32_t mant = h & 0x3FF; + uint32_t f; + if (exp == 0) { + if (mant == 0) { f = sign; } + else { + exp = 127 - 15 + 1; + while ((mant & 0x400) == 0) { mant <<= 1; exp--; } + mant &= 0x3FF; + f = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1F) { + f = sign | 0x7F800000u | (mant << 13); + } else { + f = sign | ((exp + (127 - 15)) << 23) | (mant << 13); + } + float out; __builtin_memcpy(&out, &f, 4); return out; +} + +// Dequantize one 32-element Q4_K GROUP of TILE_M weight rows into the FP32 panel, +// written in TenB [k][m] order: panel[k*TILE_M + m]. +// +// A Q4_K super-block packs 256 elements as 8 groups of 32, each with a 6-bit +// scale (sc) and 6-bit min (m), plus a super-block fp16 d and dmin: +// w = (d*sc) * nibble - (dmin*m) (nibble in 0..15, no zero point) +// kb_group is the global group index: super-block = kb_group/8, group = kb_group%8. +// Group g draws its 32 nibbles from qs[(g/2)*32 .. +31], low nibble if g even and +// high nibble if g odd (matching dequantize_q4_K_block / compute_row_dot_q4_K). +static inline void __attribute__((always_inline)) +dequant_q4_K_panel(float *panel, const char *src0_batch, + int64_t mb, int64_t kb_group, int64_t nb1_0) { + static const int32_t __attribute__((aligned(32))) scatter_idx[8] = { + 0, 64, 128, 192, 256, 320, 384, 448 // byte offsets: 8 lines apart + }; + static const int32_t __attribute__((aligned(32))) gather_idx[8] = { + 0, 1, 2, 3, 4, 5, 6, 7 // 8 consecutive bytes + }; + + const int64_t sb = kb_group >> 3; // super-block index + const int g = (int) (kb_group & 7); // group within super-block + const int64_t qoff = (int64_t)(g >> 1) * 32; + const int hi = g & 1; // high-nibble group? + + unsigned long old_mask; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" // all 8 lanes active + "flw.ps f1, (%[sidx]) \n\t" // f1 = scatter offsets + "flw.ps f2, (%[gidx]) \n\t" // f2 = gather offsets + : [ms] "=&r"(old_mask) + : [sidx] "r"(scatter_idx), [gidx] "r"(gather_idx) + : "f1", "f2" + ); + + char *pbase = (char *) panel; + for (int j = 0; j < TILE_M; ++j) { + const block_q4_K *blk = + (const block_q4_K *)(src0_batch + (mb + j) * nb1_0) + sb; + const float d = me_sw_fp16(blk->d); + const float dmin = me_sw_fp16(blk->dmin); + uint8_t sc, mm; + get_scale_min_k4(g, blk->scales, &sc, &mm); + const float dsc = d * (float) sc; + const float negmin = -(dmin * (float) mm); + uint32_t dsc_bits, nmin_bits; + __builtin_memcpy(&dsc_bits, &dsc, 4); + __builtin_memcpy(&nmin_bits, &negmin, 4); + + const uint8_t *qs = blk->qs + qoff; // 32 bytes for this group + char *col = pbase + j * 4; // column m=j of the panel + + if (hi) { + __asm__ volatile( + "fbcx.ps f3, %[dsc] \n\t" // (d*sc) in all 8 lanes + "fbcx.ps f7, %[nmin] \n\t" // -(dmin*m) in all 8 lanes + "fgb.ps f4,f2(%[q0]) \n\t fsrli.pi f5,f4,4\n\t fandi.pi f5,f5,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c0]) \n\t" + "fgb.ps f4,f2(%[q8]) \n\t fsrli.pi f5,f4,4\n\t fandi.pi f5,f5,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c8]) \n\t" + "fgb.ps f4,f2(%[q16])\n\t fsrli.pi f5,f4,4\n\t fandi.pi f5,f5,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c16])\n\t" + "fgb.ps f4,f2(%[q24])\n\t fsrli.pi f5,f4,4\n\t fandi.pi f5,f5,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c24])\n\t" + : + : [dsc] "r"(dsc_bits), [nmin] "r"(nmin_bits), + [q0] "r"(qs), [q8] "r"(qs + 8), [q16] "r"(qs + 16), [q24] "r"(qs + 24), + [c0] "r"(col), [c8] "r"(col + 8 * 64), [c16] "r"(col + 16 * 64), [c24] "r"(col + 24 * 64) + : "f3", "f4", "f5", "f7", "memory" + ); + } else { + __asm__ volatile( + "fbcx.ps f3, %[dsc] \n\t" + "fbcx.ps f7, %[nmin] \n\t" + "fgb.ps f4,f2(%[q0]) \n\t fandi.pi f5,f4,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c0]) \n\t" + "fgb.ps f4,f2(%[q8]) \n\t fandi.pi f5,f4,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c8]) \n\t" + "fgb.ps f4,f2(%[q16])\n\t fandi.pi f5,f4,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c16])\n\t" + "fgb.ps f4,f2(%[q24])\n\t fandi.pi f5,f4,15\n\t fcvt.ps.pw f5,f5,rne\n\t fmadd.ps f5,f3,f5,f7\n\t fscw.ps f5,f1(%[c24])\n\t" + : + : [dsc] "r"(dsc_bits), [nmin] "r"(nmin_bits), + [q0] "r"(qs), [q8] "r"(qs + 8), [q16] "r"(qs + 16), [q24] "r"(qs + 24), + [c0] "r"(col), [c8] "r"(col + 8 * 64), [c16] "r"(col + 16 * 64), [c24] "r"(col + 24 * 64) + : "f3", "f4", "f5", "f7", "memory" + ); + } + } + + __asm__ volatile("mova.m.x %0" :: "r"(old_mask)); +} + +// Spill / seed the FP32 C accumulator (16x16 tile in the vector register file, +// row n -> f2n[cols 0..7], f2n+1[cols 8..15]) to/from a 1 KB L2-SCP scratch. +// scratch layout: row n at byte offset n*64. Always moves all 16 rows; rows +// beyond a partial n_cur carry harmless garbage (never stored / recomputed). +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); +} + +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} + +int entry_point(struct ggml_et_binary_params *params, void *env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + // Dimensions (both harts need these for tile assignment) + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + if ((M % TILE_M) != 0) return 0; + if ((K % QK_K) != 0) return 0; + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t k_steps = K / BLOCK_K; // number of Q4_0 blocks + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE; + const int64_t local_tile_idx = local_minion; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + // L2 SCP pointers for this minion. + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // Calculate ru_n to perfectly minimize hardware waves while avoiding Consumer bottleneck. + // The pipeline is perfectly balanced at r=8. Score = waves * max(8, r). + // We find the r that minimizes Score. + int64_t best_r = 1; + int64_t min_score = INT64_MAX; + int64_t max_search_r = REUSE_MAX; + if (max_search_r > n_tiles) max_search_r = n_tiles; + + for (int64_t r = 1; r <= max_search_r; r++) { + int64_t n_groups = (n_tiles + r - 1) / r; + int64_t base_units = m_tiles * n_groups * batch_count; + int64_t waves = (base_units + MACHINE_SLOTS - 1) / MACHINE_SLOTS; + + int64_t penalty = (r > BAL) ? r : BAL; + int64_t score = waves * penalty; + +#if TIE_PREFER_LARGE + if (score <= min_score) { // largest r among equal scores = max reuse + min_score = score; + best_r = r; + } +#else + if (score < min_score) { + min_score = score; + best_r = r; + } +#endif + } + int64_t ru_n = best_r; + + // Reuse pays only when it groups >=2 N-tiles; otherwise the windowing / + // C round-trip is pure overhead, so use the one-tile-at-a-time path. + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: dequant each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t n_groups = (n_tiles + ru_n - 1) / ru_n; + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + const int64_t n_windows = (k_steps + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + dequant_q4_K_panel(cf + i * (SCP_PANEL_SIZE / 4), + src0_batch, mb, kb0 + i, nb1_0); + } + FENCE; + flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } + + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; // first N-tile + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + const int is_last = (kw == n_windows - 1); + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + if (kw > 0) c_seed(cs); + int first = (kw == 0) ? 1 : 0; + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + for (int64_t i = 0; i < kbn; ++i) { + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = (kb0 + i) * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(cf + i * (SCP_PANEL_SIZE / 4) + half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + } + + if (is_last) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } + + // ===================================================================== + // ORIGINAL path: one output tile at a time (N % TILE_N != 0). No reuse. + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { + (float *) et_shire_l2scp_local(scp_base), + (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), + }; + + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); + + dequant_q4_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_panel[buf], BLOCK_K, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + FENCE; + return 0; + } + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t chunk_id = 0; + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + int first = 1; + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + chunk_id++; + scp_wait(ready_ctr, chunk_id); + + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = kb * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + + scp_signal(consumed_ctr, chunk_id); + } + + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K.c new file mode 100644 index 000000000000..626a0a10c8f0 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K.c @@ -0,0 +1,345 @@ +//****************************************************************************** +// MUL_MAT Kernel (Q5_K weights x F32 activations -> F32) +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +// +// Structure mirrors mul_mat_Q4_K.c. Q5_K also packs 256 elements per +// super-block (16 groups of 16) with per-group int8 scales, so the same +// super-block K-tiling applies; only the per-element dequant differs and is +// delegated to Q5K_DOT() in block_ops.h. +//****************************************************************************** + +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "math_fp.h" +#include "quants.h" +#include "block_ops.h" + +#define STRIDE_M 2048 /* 32 shires x 32 minions x 2 harts */ +#define STRIDE_M_KSPLIT 1024 /* 32 shires x 32 minions (both harts share rows) */ +#define KSPLIT_MIN_K_BLOCKS 32 /* K >= 8192 elements (32 super-blocks) */ +#define KSPLIT_SMALL_ROWS_K_BLOCKS 8 /* K >= 2048 elements for very small M */ +#define KSPLIT_MAX_ROWS 8 /* max rows per minion for K-split */ +#define TILE_KB 32 /* K-tile size in Q5_K super-blocks (8192 elems) */ +#define KSPLIT_GROUP_ROWS 4 + +// Vectorized (8-wide) dot +#define Q5K_DOT(a, b, c) compute_row_dot_q5_K_vec(a, b, c) + +#ifdef ET_UBERKERNEL +static inline size_t tensor_bytes(const struct ggml_tensor* t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} +#endif + +int entry_point(struct ggml_et_binary_params* params, void* env) { + uint64_t hart_id = get_hart_id(); + +#ifdef ET_UBERKERNEL + // Uberkernel coherency: src1 (activations) may be stale in L1/L2; force + // re-read from L3/DRAM. src0 (weights) is read-only, never stale. + evict_region_past_l2(params->src1.data, tensor_bytes(¶ms->src1)); +#endif + + // Matrix dimensions + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + const int64_t ne02 = params->src0.ne[2]; + const int64_t ne03 = params->src0.ne[3]; + const int64_t ne12 = params->src1.ne[2]; + const int64_t ne13 = params->src1.ne[3]; + + // Strides (in bytes) + const size_t nb01 = params->src0.nb[1]; + const size_t nb02 = params->src0.nb[2]; + const size_t nb03 = params->src0.nb[3]; + + const size_t nb11 = params->src1.nb[1]; + const size_t nb12 = params->src1.nb[2]; + const size_t nb13 = params->src1.nb[3]; + + const size_t nbd1 = params->dst.nb[1]; + const size_t nbd2 = params->dst.nb[2]; + const size_t nbd3 = params->dst.nb[3]; + + // Q5_K super-block holds 256 elements + const int64_t K_blocks = K / QK_K; + + // Broadcasting ratios + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // K-split decision + const int64_t minion_id = hart_id >> 1; /* 0..1023 global */ + const int64_t local_minion = (hart_id >> 1) & 0x1F; /* 0..31 within shire */ + const int is_hart1 = hart_id & 1; + const int64_t rows_per_minion = (M + STRIDE_M_KSPLIT - 1) / STRIDE_M_KSPLIT; + const int64_t k_half = K_blocks / 2; + const int use_ksplit_small_rows = (rows_per_minion <= 2) + && (K_blocks >= KSPLIT_SMALL_ROWS_K_BLOCKS); + /* + * K-split when K is large enough to benefit, and either: + * - few rows (<=4): always safe, proven working + * - more rows (5-8): only if each hart's half fits in one tile, + * otherwise L1 thrashing from 2 harts x 8 rows kills performance + * + * Also allow K-split earlier for the low-M regime (<=2 rows/minion). In + * that case the simple row-striped path leaves half the machine idle, so + * using both harts on each row pays off even for moderate K. + */ + const int use_ksplit = ((K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion <= KSPLIT_MAX_ROWS) + && (rows_per_minion <= 4 || k_half <= TILE_KB)) + || use_ksplit_small_rows; + const int use_ksplit_group = !use_ksplit + && (K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion > 4) + && (rows_per_minion <= KSPLIT_MAX_ROWS); + + if (use_ksplit) { + /* Each hart processes half the K dimension */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + + /* One cache-line-aligned L2SCP slot per minion for exchange */ + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = minion_id; m < M; m += STRIDE_M_KSPLIT) { + const block_q5_K* q_row = (const block_q5_K*)(src0_ptr2 + m * nb01); + + float partial = Q5K_DOT( + q_row + k_start, b_col_base + k_start * QK_K, k_len); + + if (is_hart1) { + *l2scp_slot = partial; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + float other = *l2scp_slot; + et_sem_post(ET_BARRIER_MINION); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, partial + other); + } + } + } + } + } + } else if (use_ksplit_group) { + /* + * Grouped K-split for the 5-8 rows/minion regime. + * + * Both harts process the same 4-row group, each on half of K, and + * exchange 4 partial sums once per group instead of once per row. + */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m_base = minion_id; m_base < M; + m_base += STRIDE_M_KSPLIT * KSPLIT_GROUP_ROWS) { + const int64_t m0 = m_base; + const int64_t m1 = m0 + STRIDE_M_KSPLIT; + const int64_t m2 = m1 + STRIDE_M_KSPLIT; + const int64_t m3 = m2 + STRIDE_M_KSPLIT; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = k_len - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + if (tile_len <= 0) { + break; + } + const float* b_tile = b_col_base + (k_start + kb) * QK_K; + const int64_t row_kb = k_start + kb; + + if (m0 < M) { + s0 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m0 * nb01) + row_kb, + b_tile, tile_len); + } + if (m1 < M) { + s1 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m1 * nb01) + row_kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m2 * nb01) + row_kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m3 * nb01) + row_kb, + b_tile, tile_len); + } + } + + if (is_hart1) { + l2scp_slot[0] = s0; + l2scp_slot[1] = s1; + l2scp_slot[2] = s2; + l2scp_slot[3] = s3; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + const float p0 = l2scp_slot[0]; + const float p1 = l2scp_slot[1]; + const float p2 = l2scp_slot[2]; + const float p3 = l2scp_slot[3]; + et_sem_post(ET_BARRIER_MINION); + + float* c_base = (float*)(dst_ptr2 + n * nbd1); + if (m0 < M) atomic_store_f32((volatile float*)(c_base + m0), s0 + p0); + if (m1 < M) atomic_store_f32((volatile float*)(c_base + m1), s1 + p1); + if (m2 < M) atomic_store_f32((volatile float*)(c_base + m2), s2 + p2); + if (m3 < M) atomic_store_f32((volatile float*)(c_base + m3), s3 + p3); + } + } + } + } + } + } else if (K_blocks > TILE_KB) { + /* + * Tile-outer with scalar row groups: process up to 4 rows per + * hart sharing each B tile before advancing to the next tile. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * 4) { + const int64_t m1 = m0 + STRIDE_M; + const int64_t m2 = m0 + STRIDE_M * 2; + const int64_t m3 = m0 + STRIDE_M * 3; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = K_blocks - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + const float* b_tile = b_col_base + kb * QK_K; + + s0 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m0 * nb01) + kb, + b_tile, tile_len); + if (m1 < M) { + s1 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m1 * nb01) + kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m2 * nb01) + kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q5K_DOT( + (const block_q5_K*)(src0_ptr2 + m3 * nb01) + kb, + b_tile, tile_len); + } + } + + float* dst_base = (float*)(dst_ptr2 + n * nbd1); + atomic_store_f32((volatile float*)(dst_base + m0), s0); + if (m1 < M) atomic_store_f32((volatile float*)(dst_base + m1), s1); + if (m2 < M) atomic_store_f32((volatile float*)(dst_base + m2), s2); + if (m3 < M) atomic_store_f32((volatile float*)(dst_base + m3), s3); + } + } + } + } + } else { + /* + * Simple path for small K: one row per hart. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q5_K* q_row = (const block_q5_K*)(src0_ptr2 + m * nb01); + + float sum = Q5K_DOT(q_row, b_col_base, K_blocks); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, sum); + } + } + } + } + } + +#ifdef ET_UBERKERNEL + // Publish dst to L3/DRAM for the next uberkernel op. + FENCE; + evict_region_past_l2(params->dst.data, tensor_bytes(¶ms->dst)); + WAIT_CACHEOPS; + FENCE; +#endif + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c new file mode 100644 index 000000000000..e78e4102bef2 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c @@ -0,0 +1,535 @@ +//****************************************************************************** +// MUL_MAT Kernel +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +//****************************************************************************** + +#include +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" +#include "quants.h" +#include "math_fp.h" + +// Q5_K x F32 -> F32 MUL_MAT on the tensor (matrix) engine, TensorFMA32. +// Identical producer/consumer, tiling and tensor-engine loop to +// mul_mat_Q4_K_matrix_engine.c; only the weight dequant differs (Q5_K uses a +// 6-bit quant with a per-16-element int8 scale and no min term). +// Hart 1: dequantize Q5_K weights to FP32 into double-buffered L2 SCP. +// Hart 0: tensor engine compute (FMA, reduce, store). +// +// Two execution paths (selected at runtime by N % TILE_N): +// * REUSE path (N % TILE_N == 0): dequantize each weight K-window ONCE and +// reuse it across ru_n consecutive N-tiles, so the (producer-bound) +// dequant work is cut by ~ru_n. Partial C is round-tripped through an +// L2-SCP scratch between K-windows (the FMA C accumulator is a single fixed +// register-file tile, so multiple output tiles cannot be resident at once). +// * ORIGINAL path (N % TILE_N != 0): one output tile at a time, no reuse. + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +#define TILE_M 16 +#define TILE_N 16 +#define BLOCK_K 32 // one Q5_K group (32 elements) per panel +#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) + +// --- Reuse knobs ---------------------------------------------------------- +// REUSE_MAX caps the L2-SCP C-scratch footprint; the actual reuse factor is +// chosen at runtime as the largest value that still keeps the whole +// machine busy. KWIN is the dequant-cache depth (K-blocks per window). +#ifndef REUSE_MAX +#define REUSE_MAX 15 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per dequant window (cache depth) +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 + +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) + +// Single dequant panel: BLOCK_K k-lines x TILE_M m (FP32) = 32*64 = 2048 bytes, +// [k][m] order: panel[k*TILE_M + m]. +#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t)sizeof(float)) // 2048 + +// L2 SCP layout per minion. The REUSE path needs the larger footprint, so the +// per-minion stride uses it for both paths (mutually exclusive at runtime). +// [0 .. RU_BUF_BYTES) cache buffer 0 (KWIN panels) +// [RU_BUF_BYTES .. 2*..) cache buffer 1 (KWIN panels) +// [RU_CACHE_BYTES .. +R*1024) REUSE_MAX C-scratch tiles (16 rows*64B each) +// ready_ctr, consumed_ctr sync counters +// The ORIGINAL path reuses [0,2048) and [2048,4096) as its two panels and the +// same ready/consumed counters (which sit above the cache region). +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) + +// Software fp16->fp32 (pure integer). The hardware fcvt.ps.f16 returns wrong +// values after the attention block (shared conversion-unit state), which would +// corrupt the weight scales here; software conversion avoids that instruction +// entirely. Only the super-block d is fp16, so cost is negligible. +static inline float __attribute__((always_inline)) me_sw_fp16(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000) << 16; + uint32_t exp = (h >> 10) & 0x1F; + uint32_t mant = h & 0x3FF; + uint32_t f; + if (exp == 0) { + if (mant == 0) { f = sign; } + else { + exp = 127 - 15 + 1; + while ((mant & 0x400) == 0) { mant <<= 1; exp--; } + mant &= 0x3FF; + f = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1F) { + f = sign | 0x7F800000u | (mant << 13); + } else { + f = sign | ((exp + (127 - 15)) << 23) | (mant << 13); + } + float out; __builtin_memcpy(&out, &f, 4); return out; +} + +// EXPERIMENT: use the hardware fcvt.ps.f16 (fp16_to_fp32, math_fp.h) for the +// super-block scale instead of the software me_sw_fp16 above. Hardware fcvt was +// observed to corrupt Q4_K scales after the attention block; trying it here for +// the other K-quants. If it produces garbage, flip ME_FP16 back to me_sw_fp16. +// #define ME_FP16(h) fp16_to_fp32(h) // hardware fcvt: garbage for Q3/Q5/Q6 (fcvt-after-attention bug) +#define ME_FP16(h) me_sw_fp16(h) + +// Dequantize one 32-element Q5_K GROUP of TILE_M weight rows into the FP32 panel, +// written in TenB [k][m] order: panel[k*TILE_M + m]. +// +// A Q5_K super-block has the same affine layout as Q4_K (8 groups of 32) with an +// extra high bit per weight drawn from qh: +// w = d*sc*((nibble) + (qh_bit ? 16 : 0)) - dmin*m +// kb_group is the global 32-element group index: super-block = kb_group/8, +// group g = kb_group%8 -> qs pair p = g/2, low/high nibble ab = g%1, qh bit +// 1<<(2p+ab) (see dequantize_q5_K_block). +static inline void __attribute__((always_inline)) +dequant_q5_K_panel(float *panel, const char *src0_batch, + int64_t mb, int64_t kb_group, int64_t nb1_0) { + const int64_t sb = kb_group >> 3; + const int g = (int) (kb_group & 7); + const int p = g >> 1; // qs pair (0..3), 32-byte block + const int ab = g & 1; // 0 low nibble, 1 high nibble + const int qloff = p * 32; + const uint8_t ubit = (uint8_t) (1u << (2 * p + ab)); + + for (int j = 0; j < TILE_M; ++j) { + const block_q5_K *blk = + (const block_q5_K *)(src0_batch + (mb + j) * nb1_0) + sb; + const float d = ME_FP16(blk->d); + const float dm = ME_FP16(blk->dmin); + const uint8_t * ql = blk->qs + qloff; + const uint8_t * qh = blk->qh; + uint8_t sc, mm; + get_scale_min_k4(g, blk->scales, &sc, &mm); + const float dl = d * (float) sc; + const float ml = dm * (float) mm; + + for (int l = 0; l < 32; ++l) { + const uint8_t nib = ab ? (ql[l] >> 4) : (ql[l] & 0xF); + const int hi5 = (qh[l] & ubit) ? 16 : 0; + panel[l * TILE_M + j] = dl * (float) (nib + hi5) - ml; + } + } +} + +// Spill / seed the FP32 C accumulator (16x16 tile in the vector register file, +// row n -> f2n[cols 0..7], f2n+1[cols 8..15]) to/from a 1 KB L2-SCP scratch. +// scratch layout: row n at byte offset n*64. Always moves all 16 rows; rows +// beyond a partial n_cur carry harmless garbage (never stored / recomputed). +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); +} + +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} + +int entry_point(struct ggml_et_binary_params *params, void *env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + // Dimensions (both harts need these for tile assignment) + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + if ((M % TILE_M) != 0) return 0; + if ((K % QK_K) != 0) return 0; + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t k_steps = K / BLOCK_K; // number of 32-element groups + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE; + const int64_t local_tile_idx = local_minion; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + // L2 SCP pointers for this minion. + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // Calculate ru_n to perfectly minimize hardware waves while avoiding Consumer bottleneck. + // The pipeline is perfectly balanced at r=8. Score = waves * max(8, r). + // We find the r that minimizes Score. + int64_t best_r = 1; + int64_t min_score = INT64_MAX; + int64_t max_search_r = REUSE_MAX; + if (max_search_r > n_tiles) max_search_r = n_tiles; + + for (int64_t r = 1; r <= max_search_r; r++) { + int64_t n_groups = (n_tiles + r - 1) / r; + int64_t base_units = m_tiles * n_groups * batch_count; + int64_t waves = (base_units + MACHINE_SLOTS - 1) / MACHINE_SLOTS; + + int64_t penalty = (r > 8) ? r : 8; + int64_t score = waves * penalty; + + if (score < min_score) { + min_score = score; + best_r = r; + } + } + int64_t ru_n = best_r; + + // Reuse pays only when it groups >=2 N-tiles; otherwise the windowing / + // C round-trip is pure overhead, so use the one-tile-at-a-time path. + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: dequant each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t n_groups = (n_tiles + ru_n - 1) / ru_n; + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + const int64_t n_windows = (k_steps + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + dequant_q5_K_panel(cf + i * (SCP_PANEL_SIZE / 4), + src0_batch, mb, kb0 + i, nb1_0); + } + FENCE; + flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } + + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; // first N-tile + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + const int is_last = (kw == n_windows - 1); + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + if (kw > 0) c_seed(cs); + int first = (kw == 0) ? 1 : 0; + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + for (int64_t i = 0; i < kbn; ++i) { + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = (kb0 + i) * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(cf + i * (SCP_PANEL_SIZE / 4) + half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + } + + if (is_last) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } + + // ===================================================================== + // ORIGINAL path: one output tile at a time (N % TILE_N != 0). No reuse. + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { + (float *) et_shire_l2scp_local(scp_base), + (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), + }; + + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); + + dequant_q5_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_panel[buf], BLOCK_K, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + FENCE; + return 0; + } + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t chunk_id = 0; + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + int first = 1; + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + chunk_id++; + scp_wait(ready_ctr, chunk_id); + + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = kb * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + + scp_signal(consumed_ctr, chunk_id); + } + + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K.c new file mode 100644 index 000000000000..0a0dbb48e463 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K.c @@ -0,0 +1,347 @@ +//****************************************************************************** +// MUL_MAT Kernel (Q6_K weights x F32 activations -> F32) +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +// +// Structure mirrors mul_mat_Q4_K.c. Q6_K also packs 256 elements per +// super-block (16 groups of 16) with per-group int8 scales, so the same +// super-block K-tiling applies; only the per-element dequant differs and is +// delegated to Q6K_DOT() in block_ops.h. +//****************************************************************************** + +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "math_fp.h" +#include "quants.h" +#include "block_ops.h" + +#define STRIDE_M 2048 /* 32 shires x 32 minions x 2 harts */ +#define STRIDE_M_KSPLIT 1024 /* 32 shires x 32 minions (both harts share rows) */ +#define KSPLIT_MIN_K_BLOCKS 32 /* K >= 8192 elements (32 super-blocks) */ +#define KSPLIT_SMALL_ROWS_K_BLOCKS 8 /* K >= 2048 elements for very small M */ +#define KSPLIT_MAX_ROWS 8 /* max rows per minion for K-split */ +#define TILE_KB 32 /* K-tile size in Q6_K super-blocks (8192 elems) */ +#define KSPLIT_GROUP_ROWS 4 + +// Vectorized (8-wide) dot +// (Gather-reduction and multi-accumulator variants were tried and dropped - +// both perf-neutral; the generation dot is memory/overhead bound, not compute.) +#define Q6K_DOT(a, b, c) compute_row_dot_q6_K_vec(a, b, c) + +#ifdef ET_UBERKERNEL +static inline size_t tensor_bytes(const struct ggml_tensor* t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} +#endif + +int entry_point(struct ggml_et_binary_params* params, void* env) { + uint64_t hart_id = get_hart_id(); + +#ifdef ET_UBERKERNEL + // Uberkernel coherency: src1 (activations) may be stale in L1/L2; force + // re-read from L3/DRAM. src0 (weights) is read-only, never stale. + evict_region_past_l2(params->src1.data, tensor_bytes(¶ms->src1)); +#endif + + // Matrix dimensions + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + const int64_t ne02 = params->src0.ne[2]; + const int64_t ne03 = params->src0.ne[3]; + const int64_t ne12 = params->src1.ne[2]; + const int64_t ne13 = params->src1.ne[3]; + + // Strides (in bytes) + const size_t nb01 = params->src0.nb[1]; + const size_t nb02 = params->src0.nb[2]; + const size_t nb03 = params->src0.nb[3]; + + const size_t nb11 = params->src1.nb[1]; + const size_t nb12 = params->src1.nb[2]; + const size_t nb13 = params->src1.nb[3]; + + const size_t nbd1 = params->dst.nb[1]; + const size_t nbd2 = params->dst.nb[2]; + const size_t nbd3 = params->dst.nb[3]; + + // Q6_K super-block holds 256 elements + const int64_t K_blocks = K / QK_K; + + // Broadcasting ratios + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // K-split decision + const int64_t minion_id = hart_id >> 1; /* 0..1023 global */ + const int64_t local_minion = (hart_id >> 1) & 0x1F; /* 0..31 within shire */ + const int is_hart1 = hart_id & 1; + const int64_t rows_per_minion = (M + STRIDE_M_KSPLIT - 1) / STRIDE_M_KSPLIT; + const int64_t k_half = K_blocks / 2; + const int use_ksplit_small_rows = (rows_per_minion <= 2) + && (K_blocks >= KSPLIT_SMALL_ROWS_K_BLOCKS); + /* + * K-split when K is large enough to benefit, and either: + * - few rows (<=4): always safe, proven working + * - more rows (5-8): only if each hart's half fits in one tile, + * otherwise L1 thrashing from 2 harts x 8 rows kills performance + * + * Also allow K-split earlier for the low-M regime (<=2 rows/minion). In + * that case the simple row-striped path leaves half the machine idle, so + * using both harts on each row pays off even for moderate K. + */ + const int use_ksplit = ((K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion <= KSPLIT_MAX_ROWS) + && (rows_per_minion <= 4 || k_half <= TILE_KB)) + || use_ksplit_small_rows; + const int use_ksplit_group = !use_ksplit + && (K_blocks >= KSPLIT_MIN_K_BLOCKS) + && (rows_per_minion > 4) + && (rows_per_minion <= KSPLIT_MAX_ROWS); + + if (use_ksplit) { + /* Each hart processes half the K dimension */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + + /* One cache-line-aligned L2SCP slot per minion for exchange */ + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = minion_id; m < M; m += STRIDE_M_KSPLIT) { + const block_q6_K* q_row = (const block_q6_K*)(src0_ptr2 + m * nb01); + + float partial = Q6K_DOT( + q_row + k_start, b_col_base + k_start * QK_K, k_len); + + if (is_hart1) { + *l2scp_slot = partial; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + float other = *l2scp_slot; + et_sem_post(ET_BARRIER_MINION); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, partial + other); + } + } + } + } + } + } else if (use_ksplit_group) { + /* + * Grouped K-split for the 5-8 rows/minion regime. + * + * Both harts process the same 4-row group, each on half of K, and + * exchange 4 partial sums once per group instead of once per row. + */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + volatile float* l2scp_slot = + (volatile float*)et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m_base = minion_id; m_base < M; + m_base += STRIDE_M_KSPLIT * KSPLIT_GROUP_ROWS) { + const int64_t m0 = m_base; + const int64_t m1 = m0 + STRIDE_M_KSPLIT; + const int64_t m2 = m1 + STRIDE_M_KSPLIT; + const int64_t m3 = m2 + STRIDE_M_KSPLIT; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = k_len - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + if (tile_len <= 0) { + break; + } + const float* b_tile = b_col_base + (k_start + kb) * QK_K; + const int64_t row_kb = k_start + kb; + + if (m0 < M) { + s0 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m0 * nb01) + row_kb, + b_tile, tile_len); + } + if (m1 < M) { + s1 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m1 * nb01) + row_kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m2 * nb01) + row_kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m3 * nb01) + row_kb, + b_tile, tile_len); + } + } + + if (is_hart1) { + l2scp_slot[0] = s0; + l2scp_slot[1] = s1; + l2scp_slot[2] = s2; + l2scp_slot[3] = s3; + FENCE; + flush_to_l2((const void*)l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + const float p0 = l2scp_slot[0]; + const float p1 = l2scp_slot[1]; + const float p2 = l2scp_slot[2]; + const float p3 = l2scp_slot[3]; + et_sem_post(ET_BARRIER_MINION); + + float* c_base = (float*)(dst_ptr2 + n * nbd1); + if (m0 < M) atomic_store_f32((volatile float*)(c_base + m0), s0 + p0); + if (m1 < M) atomic_store_f32((volatile float*)(c_base + m1), s1 + p1); + if (m2 < M) atomic_store_f32((volatile float*)(c_base + m2), s2 + p2); + if (m3 < M) atomic_store_f32((volatile float*)(c_base + m3), s3 + p3); + } + } + } + } + } + } else if (K_blocks > TILE_KB) { + /* + * Tile-outer with scalar row groups: process up to 4 rows per + * hart sharing each B tile before advancing to the next tile. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * 4) { + const int64_t m1 = m0 + STRIDE_M; + const int64_t m2 = m0 + STRIDE_M * 2; + const int64_t m3 = m0 + STRIDE_M * 3; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = K_blocks - kb; + if (tile_len > TILE_KB) tile_len = TILE_KB; + const float* b_tile = b_col_base + kb * QK_K; + + s0 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m0 * nb01) + kb, + b_tile, tile_len); + if (m1 < M) { + s1 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m1 * nb01) + kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m2 * nb01) + kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += Q6K_DOT( + (const block_q6_K*)(src0_ptr2 + m3 * nb01) + kb, + b_tile, tile_len); + } + } + + float* dst_base = (float*)(dst_ptr2 + n * nbd1); + atomic_store_f32((volatile float*)(dst_base + m0), s0); + if (m1 < M) atomic_store_f32((volatile float*)(dst_base + m1), s1); + if (m2 < M) atomic_store_f32((volatile float*)(dst_base + m2), s2); + if (m3 < M) atomic_store_f32((volatile float*)(dst_base + m3), s3); + } + } + } + } + } else { + /* + * Simple path for small K: one row per hart. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char* src0_ptr3 = (const char*)params->src0.data + i03 * nb03; + const char* src1_ptr3 = (const char*)params->src1.data + i3 * nb13; + char* dst_ptr3 = (char*)params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char* src0_ptr2 = src0_ptr3 + i02 * nb02; + const char* src1_ptr2 = src1_ptr3 + i2 * nb12; + char* dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float* b_col_base = (const float*)(src1_ptr2 + n * nb11); + + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q6_K* q_row = (const block_q6_K*)(src0_ptr2 + m * nb01); + + float sum = Q6K_DOT(q_row, b_col_base, K_blocks); + + float* dst_entry = (float*)(dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float*)dst_entry, sum); + } + } + } + } + } + +#ifdef ET_UBERKERNEL + // Publish dst to L3/DRAM for the next uberkernel op. + FENCE; + evict_region_past_l2(params->dst.data, tensor_bytes(¶ms->dst)); + WAIT_CACHEOPS; + FENCE; +#endif + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c new file mode 100644 index 000000000000..1345df09453c --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c @@ -0,0 +1,538 @@ +//****************************************************************************** +// MUL_MAT Kernel +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +//****************************************************************************** + +#include +#include +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" +#include "quants.h" +#include "math_fp.h" + +// Q6_K x F32 -> F32 MUL_MAT on the tensor (matrix) engine, TensorFMA32. +// Identical producer/consumer, tiling and tensor-engine loop to +// mul_mat_Q4_K_matrix_engine.c; only the weight dequant differs (Q6_K uses a +// 6-bit quant with a per-16-element int8 scale and no min term). +// Hart 1: dequantize Q6_K weights to FP32 into double-buffered L2 SCP. +// Hart 0: tensor engine compute (FMA, reduce, store). +// +// Two execution paths (selected at runtime by N % TILE_N): +// * REUSE path (N % TILE_N == 0): dequantize each weight K-window ONCE and +// reuse it across ru_n consecutive N-tiles, so the (producer-bound) +// dequant work is cut by ~ru_n. Partial C is round-tripped through an +// L2-SCP scratch between K-windows (the FMA C accumulator is a single fixed +// register-file tile, so multiple output tiles cannot be resident at once). +// * ORIGINAL path (N % TILE_N != 0): one output tile at a time, no reuse. + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +#define TILE_M 16 +#define TILE_N 16 +#define BLOCK_K 32 // one Q6_K group (32 elements) per panel +#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) + +// --- Reuse knobs ---------------------------------------------------------- +// REUSE_MAX caps the L2-SCP C-scratch footprint; the actual reuse factor is +// chosen at runtime as the largest value that still keeps the whole +// machine busy. KWIN is the dequant-cache depth (K-blocks per window). +#ifndef REUSE_MAX +#define REUSE_MAX 15 +#endif +#ifndef KWIN +#define KWIN 16 // K-blocks per dequant window (cache depth) +#endif + +#define MACHINE_SLOTS (NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE) // 1024 + +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) + +// Single dequant panel: BLOCK_K k-lines x TILE_M m (FP32) = 32*64 = 2048 bytes, +// [k][m] order: panel[k*TILE_M + m]. +#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t)sizeof(float)) // 2048 + +// L2 SCP layout per minion. The REUSE path needs the larger footprint, so the +// per-minion stride uses it for both paths (mutually exclusive at runtime). +// [0 .. RU_BUF_BYTES) cache buffer 0 (KWIN panels) +// [RU_BUF_BYTES .. 2*..) cache buffer 1 (KWIN panels) +// [RU_CACHE_BYTES .. +R*1024) REUSE_MAX C-scratch tiles (16 rows*64B each) +// ready_ctr, consumed_ctr sync counters +// The ORIGINAL path reuses [0,2048) and [2048,4096) as its two panels and the +// same ready/consumed counters (which sit above the cache region). +#define RU_BUF_BYTES (KWIN * SCP_PANEL_SIZE) +#define RU_CACHE_BYTES (2 * RU_BUF_BYTES) +#define RU_CSCRATCH_BYTES (REUSE_MAX * 16 * 64ULL) +#define SCP_READY_OFF (RU_CACHE_BYTES + RU_CSCRATCH_BYTES) +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) + + + +// Software fp16->fp32 (pure integer). The hardware fcvt.ps.f16 returns wrong +// values after the attention block (shared conversion-unit state), which would +// corrupt the weight scales here; software conversion avoids that instruction +// entirely. Only the super-block d is fp16, so cost is negligible. +static inline float __attribute__((always_inline)) me_sw_fp16(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000) << 16; + uint32_t exp = (h >> 10) & 0x1F; + uint32_t mant = h & 0x3FF; + uint32_t f; + if (exp == 0) { + if (mant == 0) { f = sign; } + else { + exp = 127 - 15 + 1; + while ((mant & 0x400) == 0) { mant <<= 1; exp--; } + mant &= 0x3FF; + f = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1F) { + f = sign | 0x7F800000u | (mant << 13); + } else { + f = sign | ((exp + (127 - 15)) << 23) | (mant << 13); + } + float out; __builtin_memcpy(&out, &f, 4); return out; +} + +// EXPERIMENT: use the hardware fcvt.ps.f16 (fp16_to_fp32, math_fp.h) for the +// super-block scale instead of the software me_sw_fp16 above. Hardware fcvt was +// observed to corrupt Q4_K scales after the attention block; trying it here for +// the other K-quants. If it produces garbage, flip ME_FP16 back to me_sw_fp16. +// #define ME_FP16(h) fp16_to_fp32(h) // hardware fcvt: garbage for Q3/Q5/Q6 (fcvt-after-attention bug) +#define ME_FP16(h) me_sw_fp16(h) + +// Dequantize one 32-element Q6_K GROUP of TILE_M weight rows into the FP32 panel, +// written in TenB [k][m] order: panel[k*TILE_M + m]. +// +// A Q6_K super-block packs 256 elements as two 128-element chunks of four +// 32-element groups; each 6-bit weight is (ql nibble | qh 2-bit) - 32, scaled +// by a per-16-element int8 scale and the super-block fp16 d: +// w = d * scale * ((nibble | (bits << 4)) - 32) +// kb_group is the global group index: super-block = kb_group/8, group g = kb_group%8. +// Group g maps to chunk c = g/4 and sub-group sub = g%4 (see dequantize_q6_K_block). +static inline void __attribute__((always_inline)) +dequant_q6_K_panel(float *panel, const char *src0_batch, + int64_t mb, int64_t kb_group, int64_t nb1_0) { + const int64_t sb = kb_group >> 3; // super-block index + const int g = (int) (kb_group & 7); // group within super-block + const int c = g >> 2; // 128-element chunk (0 or 1) + const int sub = g & 3; // sub-group within chunk (0..3) + + const int hi = sub >> 1; // high nibble for sub 2,3 + const int qloff = 64 * c + (sub & 1) * 32; + const int qhoff = 32 * c; + const int qhshift = 2 * sub; + const int scoff = 8 * c + 2 * sub; + + for (int j = 0; j < TILE_M; ++j) { + const block_q6_K *blk = + (const block_q6_K *)(src0_batch + (mb + j) * nb1_0) + sb; + const float d = ME_FP16(blk->d); + const uint8_t * ql = blk->ql + qloff; + const uint8_t * qh = blk->qh + qhoff; + const int8_t * sc = blk->scales + scoff; + + for (int l = 0; l < 32; ++l) { + const uint8_t nib = hi ? (ql[l] >> 4) : (ql[l] & 0xF); + const int8_t q = (int8_t)(nib | (((qh[l] >> qhshift) & 3) << 4)) - 32; + const float val = d * (float) sc[l >> 4] * (float) q; + panel[l * TILE_M + j] = val; + } + } +} + +// Spill / seed the FP32 C accumulator (16x16 tile in the vector register file, +// row n -> f2n[cols 0..7], f2n+1[cols 8..15]) to/from a 1 KB L2-SCP scratch. +// scratch layout: row n at byte offset n*64. Always moves all 16 rows; rows +// beyond a partial n_cur carry harmless garbage (never stored / recomputed). +#define C_ROW_PAIR_ST(n0, n1, base) \ + __asm__ volatile("fsw.ps f" #n0 ", (%0)\n\t fsw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "memory") +#define C_ROW_PAIR_LD(n0, n1, base) \ + __asm__ volatile("flw.ps f" #n0 ", (%0)\n\t flw.ps f" #n1 ", (%1)\n\t" \ + :: "r"((base)), "r"((base) + 32) : "f" #n0, "f" #n1) + +static inline void __attribute__((always_inline)) +c_spill(char *s) { + C_ROW_PAIR_ST(0, 1, s + 0 * 64); C_ROW_PAIR_ST(2, 3, s + 1 * 64); + C_ROW_PAIR_ST(4, 5, s + 2 * 64); C_ROW_PAIR_ST(6, 7, s + 3 * 64); + C_ROW_PAIR_ST(8, 9, s + 4 * 64); C_ROW_PAIR_ST(10, 11, s + 5 * 64); + C_ROW_PAIR_ST(12, 13, s + 6 * 64); C_ROW_PAIR_ST(14, 15, s + 7 * 64); + C_ROW_PAIR_ST(16, 17, s + 8 * 64); C_ROW_PAIR_ST(18, 19, s + 9 * 64); + C_ROW_PAIR_ST(20, 21, s + 10 * 64); C_ROW_PAIR_ST(22, 23, s + 11 * 64); + C_ROW_PAIR_ST(24, 25, s + 12 * 64); C_ROW_PAIR_ST(26, 27, s + 13 * 64); + C_ROW_PAIR_ST(28, 29, s + 14 * 64); C_ROW_PAIR_ST(30, 31, s + 15 * 64); +} + +static inline void __attribute__((always_inline)) +c_seed(char *s) { + C_ROW_PAIR_LD(0, 1, s + 0 * 64); C_ROW_PAIR_LD(2, 3, s + 1 * 64); + C_ROW_PAIR_LD(4, 5, s + 2 * 64); C_ROW_PAIR_LD(6, 7, s + 3 * 64); + C_ROW_PAIR_LD(8, 9, s + 4 * 64); C_ROW_PAIR_LD(10, 11, s + 5 * 64); + C_ROW_PAIR_LD(12, 13, s + 6 * 64); C_ROW_PAIR_LD(14, 15, s + 7 * 64); + C_ROW_PAIR_LD(16, 17, s + 8 * 64); C_ROW_PAIR_LD(18, 19, s + 9 * 64); + C_ROW_PAIR_LD(20, 21, s + 10 * 64); C_ROW_PAIR_LD(22, 23, s + 11 * 64); + C_ROW_PAIR_LD(24, 25, s + 12 * 64); C_ROW_PAIR_LD(26, 27, s + 13 * 64); + C_ROW_PAIR_LD(28, 29, s + 14 * 64); C_ROW_PAIR_LD(30, 31, s + 15 * 64); +} + +int entry_point(struct ggml_et_binary_params *params, void *env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + // Dimensions (both harts need these for tile assignment) + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + if ((M % TILE_M) != 0) return 0; + if ((K % QK_K) != 0) return 0; + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t k_steps = K / BLOCK_K; // number of 32-element groups + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE; + const int64_t local_tile_idx = local_minion; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t my_start = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + // L2 SCP pointers for this minion. + const uint64_t scp_base = local_minion * SCP_PER_MINION; + volatile uint32_t *ready_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // Calculate ru_n to perfectly minimize hardware waves while avoiding Consumer bottleneck. + // The pipeline is perfectly balanced at r=8. Score = waves * max(8, r). + // We find the r that minimizes Score. + int64_t best_r = 1; + int64_t min_score = INT64_MAX; + int64_t max_search_r = REUSE_MAX; + if (max_search_r > n_tiles) max_search_r = n_tiles; + + for (int64_t r = 1; r <= max_search_r; r++) { + int64_t n_groups = (n_tiles + r - 1) / r; + int64_t base_units = m_tiles * n_groups * batch_count; + int64_t waves = (base_units + MACHINE_SLOTS - 1) / MACHINE_SLOTS; + + int64_t penalty = (r > 8) ? r : 8; + int64_t score = waves * penalty; + + if (score < min_score) { + min_score = score; + best_r = r; + } + } + int64_t ru_n = best_r; + + // Reuse pays only when it groups >=2 N-tiles; otherwise the windowing / + // C round-trip is pure overhead, so use the one-tile-at-a-time path. + const int reuse_ok = (ru_n >= 2); + + // ===================================================================== + // REUSE path: dequant each K-window once, reuse across ru_n N-tiles. + // ===================================================================== + if (reuse_ok) { + char *cache_buf[2] = { + (char *) et_shire_l2scp_local(scp_base), + (char *) et_shire_l2scp_local(scp_base + RU_BUF_BYTES), + }; + char *cscratch = (char *) et_shire_l2scp_local(scp_base + RU_CACHE_BYTES); + + const int64_t n_groups = (n_tiles + ru_n - 1) / ru_n; + const int64_t units_pb = m_tiles * n_groups; + const int64_t base_units = units_pb * batch_count; + const int64_t n_windows = (k_steps + KWIN - 1) / KWIN; + + // ----- Hart 1: producer ----- + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t wid = 0; + + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + if (wid >= 2) scp_wait(consumed_ctr, wid - 1); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + + float *cf = (float *) cache_buf[buf]; + for (int64_t i = 0; i < kbn; ++i) { + dequant_q6_K_panel(cf + i * (SCP_PANEL_SIZE / 4), + src0_batch, mb, kb0 + i, nb1_0); + } + FENCE; + flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + WAIT_CACHEOPS; + + wid++; + scp_signal(ready_ctr, wid); + } + } + FENCE; + return 0; + } + + // ----- Hart 0: consumer ----- + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t wid = 0; + for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { + const int64_t batch_idx = unit / units_pb; + const int64_t unit_in_b = unit % units_pb; + const int64_t g_idx = unit_in_b / m_tiles; + const int64_t mb_idx = unit_in_b % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb_base_t = g_idx * ru_n; // first N-tile + int64_t r_count = n_tiles - nb_base_t; + if (r_count > ru_n) r_count = ru_n; + + for (int64_t kw = 0; kw < n_windows; ++kw) { + const int buf = wid & 1; + wid++; + scp_wait(ready_ctr, wid); + + const int64_t kb0 = kw * KWIN; + const int64_t kbn = (kb0 + KWIN <= k_steps) ? KWIN : (k_steps - kb0); + const int is_last = (kw == n_windows - 1); + float *cf = (float *) cache_buf[buf]; + + for (int64_t r = 0; r < r_count; ++r) { + const int64_t nb = (nb_base_t + r) * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + char *cs = cscratch + r * (16 * 64); + + if (kw > 0) c_seed(cs); + int first = (kw == 0) ? 1 : 0; + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + for (int64_t i = 0; i < kbn; ++i) { + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = (kb0 + i) * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(cf + i * (SCP_PANEL_SIZE / 4) + half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + } + + if (is_last) { + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } else { + c_spill(cs); + } + } + scp_signal(consumed_ctr, wid); + } + } + FENCE; + return 0; + } + + // ===================================================================== + // ORIGINAL path: one output tile at a time (N % TILE_N != 0). No reuse. + // ===================================================================== + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + float *scp_panel[2] = { + (float *) et_shire_l2scp_local(scp_base), + (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), + }; + + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + uint32_t chunk_id = 0; + + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + if (chunk_id >= 2) scp_wait(consumed_ctr, chunk_id - 1); + + dequant_q6_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_panel[buf], BLOCK_K, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + FENCE; + return 0; + } + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; + + uint32_t chunk_id = 0; + for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + if (n_cur == 4) { + static const float __attribute__((aligned(64))) zero_line[16] = {0}; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, + (uint64_t) zero_line, 0, 0, 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + int first = 1; + for (int64_t kb = 0; kb < k_steps; ++kb) { + int buf = chunk_id & 1; + chunk_id++; + scp_wait(ready_ctr, chunk_id); + + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = kb * BLOCK_K + half * FMA_K; + tensor_load( + false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t)(src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), + 0, n_cur - 1, (uint64_t) nb1_1, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_load_setup_b( + false, + (uint64_t)(scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), + FMA_K - 1, 64, 1); + + tensor_fma( + false, 3, arows_fma, FMA_K - 1, 0, + false, false, false, true, + B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + + scp_signal(consumed_ctr, chunk_id); + } + + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), + 0, (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c index ad21a3ee043a..930d764ca6e3 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c @@ -335,59 +335,47 @@ int entry_point(struct ggml_et_mm_q8_params * params, void * env) { * When `nb01` is 32-byte aligned, every row has the same block-alignment * pattern. That lets us compute two rows together and reuse each loaded * B chunk across both rows instead of reloading it in a second dot call. + * + * For generation (M=1..4), M-split leaves most harts idle (only 1-4 + * active out of 2048). N-split distributes the N dimension across harts + * instead, keeping all N harts busy — a 100-2000x utilization gain. */ - for (int64_t i3 = 0; i3 < ne13; i3++) { - const int64_t i03 = i3 / r3; - const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; - const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; - char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; - const char * bias_ptr3 = bias_base ? bias_base + i3 * nbb3 : (const char *) 0; - - for (int64_t i2 = 0; i2 < ne12; i2++) { - const int64_t i02 = i2 / r2; - const char * src0_ptr2 = src0_ptr3 + i02 * nb02; - const char * src1_ptr2 = src1_ptr3 + i2 * nb12; - char * dst_ptr2 = dst_ptr3 + i2 * nbd2; - const char * bias_ptr2 = bias_ptr3 ? bias_ptr3 + i2 * nbb2 : (const char *) 0; - - for (int64_t n = 0; n < N; n++) { - const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); - const float * bias_n = bias_ptr2 ? (const float *) (bias_ptr2 + n * nbb1) : (const float *) 0; - q8_dot_state q8_state; + const int use_n_split = (M <= 4 && N > M && N >= 32); + if (use_n_split) { + /* + * N-split: distribute output columns across harts. + * Each hart processes ALL rows (M) for its subset of columns. + * This keeps many harts active during generation where M=1. + */ + const int64_t total_harts = 2048; + const int64_t n_per_hart = (N + total_harts - 1) / total_harts; + const int64_t n_start = hart_id * n_per_hart; + const int64_t n_end = (n_start + n_per_hart < N) ? (n_start + n_per_hart) : N; + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; + const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; + char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; + const char * bias_ptr3 = bias_base ? bias_base + i3 * nbb3 : (const char *) 0; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char * src0_ptr2 = src0_ptr3 + i02 * nb02; + const char * src1_ptr2 = src1_ptr3 + i2 * nb12; + char * dst_ptr2 = dst_ptr3 + i2 * nbd2; + const char * bias_ptr2 = bias_ptr3 ? bias_ptr3 + i2 * nbb2 : (const char *) 0; + + q8_dot_state q8_state; q8_dot_begin(&q8_state); - if (use_simple_x2) { - for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * SIMPLE_X2_ROWS) { - const int64_t m1 = m0 + STRIDE_M; - const block_q8_0 * q_row0 = (const block_q8_0 *) (src0_ptr2 + m0 * nb01); - - if (m1 < M) { - const block_q8_0 * q_row1 = (const block_q8_0 *) (src0_ptr2 + m1 * nb01); - float s0, s1; - q8_dot_compute_x2_aligned(q_row0, q_row1, b_col_base, K_blocks, &s0, &s1); + for (int64_t n = n_start; n < n_end; n++) { + const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); + const float * bias_n = bias_ptr2 ? (const float *) (bias_ptr2 + n * nbb1) : (const float *) 0; - float * dst0 = (float *) (dst_ptr2 + n * nbd1 + m0 * sizeof(float)); - float * dst1 = (float *) (dst_ptr2 + n * nbd1 + m1 * sizeof(float)); - if (bias_n) { - s0 += bias_n[m0]; - s1 += bias_n[m1]; - } - atomic_store_f32((volatile float *) dst0, s0); - atomic_store_f32((volatile float *) dst1, s1); - } else { - float sum = q8_dot_compute(q_row0, b_col_base, K_blocks); - float * dst = (float *) (dst_ptr2 + n * nbd1 + m0 * sizeof(float)); - if (bias_n) { - sum += bias_n[m0]; - } - atomic_store_f32((volatile float *) dst, sum); - } - } - } else { - for (int64_t m = hart_id; m < M; m += STRIDE_M) { + for (int64_t m = 0; m < M; m++) { const block_q8_0 * q_row = (const block_q8_0 *) (src0_ptr2 + m * nb01); - - float sum = q8_dot_compute(q_row, b_col_base, K_blocks); + float sum = q8_dot_compute(q_row, b_col_base, K_blocks); float * dst_entry = (float *) (dst_ptr2 + n * nbd1 + m * sizeof(float)); if (bias_n) { @@ -400,6 +388,72 @@ int entry_point(struct ggml_et_mm_q8_params * params, void * env) { q8_dot_end(&q8_state); } } + } else { + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; + const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; + char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; + const char * bias_ptr3 = bias_base ? bias_base + i3 * nbb3 : (const char *) 0; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char * src0_ptr2 = src0_ptr3 + i02 * nb02; + const char * src1_ptr2 = src1_ptr3 + i2 * nb12; + char * dst_ptr2 = dst_ptr3 + i2 * nbd2; + const char * bias_ptr2 = bias_ptr3 ? bias_ptr3 + i2 * nbb2 : (const char *) 0; + + for (int64_t n = 0; n < N; n++) { + const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); + const float * bias_n = bias_ptr2 ? (const float *) (bias_ptr2 + n * nbb1) : (const float *) 0; + q8_dot_state q8_state; + q8_dot_begin(&q8_state); + + if (use_simple_x2) { + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * SIMPLE_X2_ROWS) { + const int64_t m1 = m0 + STRIDE_M; + const block_q8_0 * q_row0 = (const block_q8_0 *) (src0_ptr2 + m0 * nb01); + + if (m1 < M) { + const block_q8_0 * q_row1 = (const block_q8_0 *) (src0_ptr2 + m1 * nb01); + float s0, s1; + q8_dot_compute_x2_aligned(q_row0, q_row1, b_col_base, K_blocks, &s0, &s1); + + float * dst0 = (float *) (dst_ptr2 + n * nbd1 + m0 * sizeof(float)); + float * dst1 = (float *) (dst_ptr2 + n * nbd1 + m1 * sizeof(float)); + if (bias_n) { + s0 += bias_n[m0]; + s1 += bias_n[m1]; + } + atomic_store_f32((volatile float *) dst0, s0); + atomic_store_f32((volatile float *) dst1, s1); + } else { + float sum = q8_dot_compute(q_row0, b_col_base, K_blocks); + float * dst = (float *) (dst_ptr2 + n * nbd1 + m0 * sizeof(float)); + if (bias_n) { + sum += bias_n[m0]; + } + atomic_store_f32((volatile float *) dst, sum); + } + } + } else { + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q8_0 * q_row = (const block_q8_0 *) (src0_ptr2 + m * nb01); + + float sum = q8_dot_compute(q_row, b_col_base, K_blocks); + + float * dst_entry = (float *) (dst_ptr2 + n * nbd1 + m * sizeof(float)); + if (bias_n) { + sum += bias_n[m]; + } + atomic_store_f32((volatile float *) dst_entry, sum); + } + } + + q8_dot_end(&q8_state); + } + } + } } } diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c index 2aab87ad5e52..49ed9700a4ee 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c @@ -1,10 +1,9 @@ +#include +#include #include "ggml_tensor.h" #include "platform.h" #include "tensor.h" -#include -#include - // FP16 x FP16 -> FP32 MUL_MAT with hart 1 B-panel packing // // Hart 0: tensor engine (load A, load B from SCP, FMA, reduce, store) @@ -34,28 +33,9 @@ typedef uint16_t et_fp16_t; // [2048..2111] ready counter (hart1 -> hart0, own cache line) // [2112..2175] consumed counter (hart0 -> hart1, own cache line) #define SCP_BPANEL_SIZE (16 * 32 * sizeof(et_fp16_t)) // 1024 bytes -#define SCP_READY_OFF (2 * SCP_BPANEL_SIZE) // 2048 -#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 2112 -#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 2176 - -// Signal a counter value to the other hart via L2 SCP. -static inline void __attribute__((always_inline)) scp_signal(volatile uint32_t * flag, uint32_t value) { - *flag = value; - FENCE; - evict_to_l2((const void *) flag, 1, 64); - WAIT_CACHEOPS; -} - -// Wait for a counter in L2 SCP to reach the expected value. -static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * flag, uint32_t expected) { - while (1) { - evict_to_l2((const void *) flag, 1, 64); - WAIT_CACHEOPS; - if (*flag >= expected) { - return; - } - } -} +#define SCP_READY_OFF (2 * SCP_BPANEL_SIZE) // 2048 +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 2112 +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 2176 /** * Build the interleaved B panel that TensorFMA16A32 expects (vectorized). @@ -66,12 +46,14 @@ static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * f * * Uses fsch.ps scatter store: load 8 pairs per row, scatter to 8 output lines. */ -static inline void __attribute__((always_inline)) pack_b_interleaved(et_fp16_t * out, - const char * src0_batch, - int64_t mb, - int64_t kb, - int64_t nb1_0) { - static const int32_t __attribute__((aligned(32))) scatter_idx[8] = { 0, 64, 128, 192, 256, 320, 384, 448 }; +static inline void __attribute__((always_inline)) +pack_b_interleaved(et_fp16_t *out, + const char *src0_batch, + int64_t mb, int64_t kb, int64_t nb1_0) +{ + static const int32_t __attribute__((aligned(32))) scatter_idx[8] = { + 0, 64, 128, 192, 256, 320, 384, 448 + }; unsigned long old_mask; __asm__ volatile( @@ -80,11 +62,13 @@ static inline void __attribute__((always_inline)) pack_b_interleaved(et_fp16_t * "flw.ps f1, 0(%[idx]) \n\t" : [ms] "=&r"(old_mask) : [idx] "r"(scatter_idx) - : "f1"); + : "f1" + ); for (int j = 0; j < TILE_M; ++j) { - const et_fp16_t * row = (const et_fp16_t *) (src0_batch + (mb + j) * nb1_0) + kb; - char * dst = (char *) out + j * 4; + const et_fp16_t *row = + (const et_fp16_t *)(src0_batch + (mb + j) * nb1_0) + kb; + char *dst = (char *)out + j * 4; __asm__ volatile( "flw.ps f2, 0(%[src]) \n\t" @@ -92,25 +76,30 @@ static inline void __attribute__((always_inline)) pack_b_interleaved(et_fp16_t * "fscw.ps f2, f1(%[d0]) \n\t" "fscw.ps f3, f1(%[d1]) \n\t" : - : [src] "r"(row), [d0] "r"(dst), [d1] "r"(dst + 512) - : "f2", "f3", "memory"); + : [src] "r"(row), + [d0] "r"(dst), + [d1] "r"(dst + 512) + : "f2", "f3", "memory" + ); } - __asm__ volatile("mova.m.x %[ms] \n\t" : : [ms] "r"(old_mask)); + __asm__ volatile( + "mova.m.x %[ms] \n\t" + : + : [ms] "r"(old_mask) + ); } -int entry_point(struct ggml_et_binary_params * params, void * env) { +int entry_point(struct ggml_et_binary_params *params, void *env) { (void) env; uint64_t hart_id = get_hart_id(); uint64_t shire_id = get_shire_id(); - if (shire_id >= NUM_COMPUTE_SHIRES) { - return 0; - } + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; - const int is_hart1 = hart_id & 1; - uint64_t local_minion = (hart_id >> 1) & 0x1F; + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; // Dimensions (both harts need these for tile assignment) const int64_t K = params->src0.ne[0]; @@ -129,31 +118,27 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const int64_t nb1_d = params->dst.nb[1]; const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; - const char * src0_base = (const char *) params->src0.data; - const char * src1_base = (const char *) params->src1.data; - char * dst_base = (char *) params->dst.data; + const char *src0_base = (const char *) params->src0.data; + const char *src1_base = (const char *) params->src1.data; + char *dst_base = (char *) params->dst.data; - if ((M % TILE_M) != 0) { - return 0; - } - if ((K % TILE_K) != 0) { - return 0; - } + if ((M % TILE_M) != 0) return 0; + if ((K % TILE_K) != 0) return 0; - const int64_t m_tiles = M / TILE_M; - const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; const int64_t batch_count = ne2_1 * ne3_1; - const int64_t base_tiles = m_tiles * n_tiles * batch_count; + const int64_t base_tiles = m_tiles * n_tiles * batch_count; const int64_t r2 = ne2_1 / ne2_0; const int64_t r3 = ne3_1 / ne3_0; const int64_t total_harts = NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE; - const int64_t k_steps = K / TILE_K; + const int64_t k_steps = K / TILE_K; int64_t k_splits = 1; if (base_tiles < total_harts) { - k_splits = (total_harts + base_tiles - 1) / base_tiles; + k_splits = (total_harts + base_tiles - 1) / base_tiles; int64_t ks = 1; while (ks * 2 <= k_splits && ks * 2 <= 32 && k_steps % (ks * 2) == 0) { ks *= 2; @@ -162,22 +147,24 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { } const int64_t tiles_per_shire = MINIONS_PER_SHIRE / k_splits; - const int64_t k_split = local_minion % k_splits; - const int64_t local_tile_idx = local_minion / k_splits; - const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t k_split = local_minion % k_splits; + const int64_t local_tile_idx = local_minion / k_splits; + const int64_t tiles_stride = (int64_t)NUM_COMPUTE_SHIRES * tiles_per_shire; const int64_t k_steps_per_split = k_steps / k_splits; - const int64_t k_start = k_split * k_steps_per_split * TILE_K; - const int64_t k_end = k_start + k_steps_per_split * TILE_K; + const int64_t k_start = k_split * k_steps_per_split * TILE_K; + const int64_t k_end = k_start + k_steps_per_split * TILE_K; // L2 SCP pointers for this minion's double-buffered panels + sync - uint64_t scp_base = local_minion * SCP_PER_MINION; - et_fp16_t * scp_bp[2] = { - (et_fp16_t *) et_shire_l2scp_local(scp_base), - (et_fp16_t *) et_shire_l2scp_local(scp_base + SCP_BPANEL_SIZE), + uint64_t scp_base = local_minion * SCP_PER_MINION; + et_fp16_t *scp_bp[2] = { + (et_fp16_t *)et_shire_l2scp_local(scp_base), + (et_fp16_t *)et_shire_l2scp_local(scp_base + SCP_BPANEL_SIZE), }; - volatile uint32_t * ready_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); - volatile uint32_t * consumed_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + volatile uint32_t *ready_ctr = + (volatile uint32_t *)et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t *consumed_ctr = + (volatile uint32_t *)et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); // ================================================================ // Hart 1: B-panel packer @@ -189,8 +176,10 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { uint32_t chunk_id = 0; - for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; + for (int64_t tile = (int64_t)shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; const int64_t batch_idx = tile / tiles_per_batch; const int64_t tile_in_batch = tile % tiles_per_batch; @@ -202,8 +191,8 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const int64_t i2_0 = i2 / r2; const int64_t i3_0 = i3 / r3; - const char * src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; - const int64_t mb = mb_idx * TILE_M; + const char *src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { int buf = chunk_id & 1; @@ -231,7 +220,7 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { // ================================================================ // Hart 0: tensor engine compute // ================================================================ - uint64_t my_minion_id = get_minion_id(); + uint64_t my_minion_id = get_minion_id(); const uint64_t group_base_global = my_minion_id - k_split; setup_cache_scp(); @@ -241,15 +230,17 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { CLEAR_TENSOR_ERROR; // Evict any stale L1D copies of sync counters - evict_to_l2((const void *) ready_ctr, 1, 64); + evict_to_l2((const void *)ready_ctr, 1, 64); WAIT_CACHEOPS; - evict_to_l2((const void *) consumed_ctr, 1, 64); + evict_to_l2((const void *)consumed_ctr, 1, 64); WAIT_CACHEOPS; uint32_t chunk_id = 0; - for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; + for (int64_t tile = (int64_t)shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; const int64_t batch_idx = tile / tiles_per_batch; const int64_t tile_in_batch = tile % tiles_per_batch; @@ -257,14 +248,14 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const int64_t nb_idx = tile_in_batch / m_tiles; const int64_t mb_idx = tile_in_batch % m_tiles; - const int64_t i3 = batch_idx / ne2_1; - const int64_t i2 = batch_idx % ne2_1; + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; - const char * src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; - char * dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + const char *src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char *dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; - const int64_t mb = mb_idx * TILE_M; - const int64_t nb = nb_idx * TILE_N; + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); // Set tensor_mask for partial N tiles @@ -277,23 +268,54 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { int buf = chunk_id & 1; // Start loading A from DRAM (overlaps with waiting for hart 1) - tensor_load((n_cur < TILE_N), false, A_L1_START, TENSOR_LOAD_PLAIN, 0, - (uint64_t) (src1_batch + nb * nb1_1 + kb * (int64_t) sizeof(et_fp16_t)), 0, n_cur - 1, - (uint64_t) nb1_1, 0); + tensor_load( + (n_cur < TILE_N), false, + A_L1_START, + TENSOR_LOAD_PLAIN, + 0, + (uint64_t)(src1_batch + nb * nb1_1 + kb * (int64_t)sizeof(et_fp16_t)), + 0, + n_cur - 1, + (uint64_t)nb1_1, + 0 + ); // Wait for hart 1 to finish packing this chunk chunk_id++; scp_wait(ready_ctr, chunk_id); // Load B from L2 SCP (hart 1 already flushed it) - tensor_load(false, false, B_L1_START, TENSOR_LOAD_PLAIN, 0, (uint64_t) scp_bp[buf], 0, 15, 64, 1); + tensor_load( + false, false, + B_L1_START, + TENSOR_LOAD_PLAIN, + 0, + (uint64_t)scp_bp[buf], + 0, + 15, + 64, + 1 + ); tensor_wait(TENSOR_LOAD_WAIT_0); tensor_wait(TENSOR_LOAD_WAIT_1); // TensorFMA16A32 - tensor_fma((n_cur < TILE_N), 3, n_cur - 1, 15, 0, false, false, false, false, B_L1_START, A_L1_START, - TENSOR_FMA_OP_FP16, (kb == k_start)); + tensor_fma( + (n_cur < TILE_N), + 3, + n_cur - 1, + 15, + 0, + false, + false, + false, + false, + B_L1_START, + A_L1_START, + TENSOR_FMA_OP_FP16, + (kb == k_start) + ); tensor_wait(TENSOR_FMA_WAIT); @@ -303,23 +325,33 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { // K-split ring reduce if (k_splits > 1) { - const uint64_t num_regs = (uint64_t) n_cur * 2; + const uint64_t num_regs = (uint64_t)n_cur * 2; if (k_split > 0) { - tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, group_base_global + k_split - 1); + tensor_reduce_recv( + 0, TENSOR_REDUCE_OP_FADD, + num_regs, + group_base_global + k_split - 1 + ); tensor_wait(TENSOR_REDUCE_WAIT); } if (k_split < k_splits - 1) { - tensor_reduce_send(0, num_regs, group_base_global + k_split + 1); + tensor_reduce_send( + 0, num_regs, + group_base_global + k_split + 1 + ); tensor_wait(TENSOR_REDUCE_WAIT); } } // Store FP32 result tile if (k_split == k_splits - 1) { - tensor_store(0, 0, 3, n_cur - 1, (uint64_t) (dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), 0, - (uint64_t) nb1_d); + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * (int64_t)sizeof(float)), + 0, (uint64_t)nb1_d + ); tensor_wait(TENSOR_STORE_WAIT); } } diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c index b2b61d519672..4d14769f5a66 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c @@ -1,10 +1,9 @@ +#include +#include #include "ggml_tensor.h" #include "platform.h" #include "tensor.h" -#include -#include - /* * F32 Matrix Multiply for ET-SoC-1 — TensorFMA32. * @@ -19,22 +18,17 @@ #define TILE_M 16 /* ── Tuning knobs ───────────────────────────────────────────────────── */ -#define TILE_N 16 -#define CACHEOP_MAX 0 -#define REP_RATE 0 - +#define TILE_N 16 +#define CACHEOP_MAX 0 +#define REP_RATE 0 /* ─────────────────────────────────────────────────────────────────── */ -int entry_point(struct ggml_et_binary_params * params, void * env) { - uint64_t hart_id = get_hart_id(); +int entry_point(struct ggml_et_binary_params* params, void* env) { + uint64_t hart_id = get_hart_id(); uint64_t shire_id = get_shire_id(); - if (shire_id >= NUM_COMPUTE_SHIRES) { - return 0; - } - if (hart_id & 1) { - return 0; - } + if (shire_id >= NUM_COMPUTE_SHIRES) return 0; + if (hart_id & 1) return 0; uint64_t local_minion = (hart_id >> 1) & 0x1F; uint64_t my_minion_id = get_minion_id(); @@ -51,11 +45,11 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { const int64_t nb1_1 = params->src1.nb[1]; const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; const int64_t nb1_d = params->dst.nb[1]; - const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; - const char * src0_base = (const char *) params->src0.data; - const char * src1_base = (const char *) params->src1.data; - char * dst_base = (char *) params->dst.data; + const char* src0_base = (const char*)params->src0.data; + const char* src1_base = (const char*)params->src1.data; + char* dst_base = (char*)params->dst.data; setup_cache_scp(); #if CACHEOP_MAX > 0 || REP_RATE > 0 @@ -63,19 +57,19 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { #endif CLEAR_TENSOR_ERROR; - const int64_t m_tiles = M / TILE_M; - const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; const int64_t batch_count = ne2_1 * ne3_1; - const int64_t base_tiles = m_tiles * n_tiles * batch_count; + const int64_t base_tiles = m_tiles * n_tiles * batch_count; const int64_t r2 = ne2_1 / ne2_0; const int64_t r3 = ne3_1 / ne3_0; const int64_t total_harts = NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE; - const int64_t k_steps = K / TILE_K; - int64_t k_splits = 1; + const int64_t k_steps = K / TILE_K; + int64_t k_splits = 1; if (base_tiles < total_harts) { - k_splits = (total_harts + base_tiles - 1) / base_tiles; + k_splits = (total_harts + base_tiles - 1) / base_tiles; int64_t ks = 1; while (ks * 2 <= k_splits && ks * 2 <= 32 && k_steps % (ks * 2) == 0) { ks *= 2; @@ -84,68 +78,88 @@ int entry_point(struct ggml_et_binary_params * params, void * env) { } const int64_t tiles_per_shire = MINIONS_PER_SHIRE / k_splits; - const int64_t k_split = local_minion % k_splits; - const int64_t local_tile_idx = local_minion / k_splits; - const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + const int64_t k_split = local_minion % k_splits; + const int64_t local_tile_idx = local_minion / k_splits; + const int64_t tiles_stride = (int64_t)NUM_COMPUTE_SHIRES * tiles_per_shire; const int64_t k_steps_per_split = k_steps / k_splits; - const int64_t k_start = k_split * k_steps_per_split * TILE_K; - const int64_t k_end = k_start + k_steps_per_split * TILE_K; + const int64_t k_start = k_split * k_steps_per_split * TILE_K; + const int64_t k_end = k_start + k_steps_per_split * TILE_K; const uint64_t group_base_global = my_minion_id - k_split; - for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; + for (int64_t tile = (int64_t)shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + tile < base_tiles; tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; - const int64_t batch_idx = tile / tiles_per_batch; - const int64_t tile_in_batch = tile % tiles_per_batch; - const int64_t nb_idx = tile_in_batch / m_tiles; - const int64_t mb_idx = tile_in_batch % m_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; const int64_t i3 = batch_idx / ne2_1; const int64_t i2 = batch_idx % ne2_1; const int64_t i2_0 = i2 / r2; const int64_t i3_0 = i3 / r3; - const char * src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; - const char * src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; - char * dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + const char* src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const char* src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char* dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; - const int64_t mb = mb_idx * TILE_M; - const int64_t nb = nb_idx * TILE_N; + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { - tensor_load(false, false, 0, 0, 0, (uint64_t) (src1_batch + nb * nb1_1 + kb * sizeof(float)), 0, n_cur - 1, - (uint64_t) nb1_1, 0); - tensor_load(false, false, TILE_K, 7, 0, (uint64_t) (src0_batch + mb * nb1_0 + kb * sizeof(float)), 0, - TILE_K - 1, (uint64_t) nb1_0, 1); + tensor_load( + false, false, 0, 0, 0, + (uint64_t)(src1_batch + nb * nb1_1 + kb * sizeof(float)), + 0, n_cur - 1, (uint64_t)nb1_1, 0 + ); + + tensor_load( + false, false, TILE_K, 7, 0, + (uint64_t)(src0_batch + mb * nb1_0 + kb * sizeof(float)), + 0, TILE_K - 1, (uint64_t)nb1_0, 1 + ); tensor_wait(TENSOR_LOAD_WAIT_0); tensor_wait(TENSOR_LOAD_WAIT_1); - tensor_fma(false, 3, n_cur - 1, TILE_K - 1, 0, false, false, false, false, TILE_K, 0, 0, (kb == k_start)); + tensor_fma( + false, 3, n_cur - 1, TILE_K - 1, 0, + false, false, false, false, + TILE_K, 0, 0, + (kb == k_start) + ); tensor_wait(TENSOR_FMA_WAIT); } if (k_splits > 1) { - const uint64_t num_regs = (uint64_t) n_cur * 2; + const uint64_t num_regs = (uint64_t)n_cur * 2; if (k_split > 0) { - tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, group_base_global + k_split - 1); + tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, + num_regs, + group_base_global + k_split - 1); tensor_wait(TENSOR_REDUCE_WAIT); } if (k_split < k_splits - 1) { - tensor_reduce_send(0, num_regs, group_base_global + k_split + 1); + tensor_reduce_send(0, num_regs, + group_base_global + k_split + 1); tensor_wait(TENSOR_REDUCE_WAIT); } } if (k_split == k_splits - 1) { - tensor_store(0, 0, 3, n_cur - 1, (uint64_t) (dst_batch + nb * nb1_d + mb * sizeof(float)), 0, - (uint64_t) nb1_d); + tensor_store( + 0, 0, 3, n_cur - 1, + (uint64_t)(dst_batch + nb * nb1_d + mb * sizeof(float)), + 0, (uint64_t)nb1_d + ); tensor_wait(TENSOR_STORE_WAIT); } } diff --git a/ggml/src/ggml-et/et-kernels/src/platform.h b/ggml/src/ggml-et/et-kernels/src/platform.h index cbec4c98d741..60efb0b6223f 100644 --- a/ggml/src/ggml-et/et-kernels/src/platform.h +++ b/ggml/src/ggml-et/et-kernels/src/platform.h @@ -13,6 +13,8 @@ #include "etsoc/isa/hart.h" #include +#include "etsoc/isa/hart.h" +#include "etsoc/common/utils.h" #define SOC_MINIONS_PER_SHIRE 32 #define NUM_HARTS_PER_MINION 2 @@ -32,13 +34,14 @@ typedef struct { // Production implementations (like libgcc's __ctzdi2) use optimized bit manipulation // algorithms with lookup tables and parallel bit operations for O(log n) performance. static inline int manual_ctzll(uint64_t x) { - if (x == 0) return 64; - int count = 0; - while ((x & 1) == 0) { - x >>= 1; - count++; - } - return count; + // if (x == 0) return 64; + // int count = 0; + // while ((x & 1) == 0) { + // x >>= 1; + // count++; + // } + // return count; + return 0; } // Manual implementation of population count for bare metal environment @@ -46,12 +49,13 @@ static inline int manual_ctzll(uint64_t x) { // Production implementations (like libgcc's __popcountdi2) use optimized bit-parallel // algorithms with magic constants and bit manipulation tricks for O(1) performance. static inline int manual_popcountll(uint64_t x) { - int count = 0; - while (x) { - count += x & 1; - x >>= 1; - } - return count; + // int count = 0; + // while (x) { + // count += x & 1; + // x >>= 1; + // } + // return count; + return 32; } // Binary GCD (Stein's algorithm) — avoids expensive 64-bit division/remainder. @@ -140,12 +144,98 @@ static inline void atomic_store_f32(volatile float * addr, float value) { et_global_swap_w(addr, *(uint32_t *) &value); } -static inline void atomic_add_f32(volatile float * addr, float value) { - et_global_add_w(addr, *(uint32_t *) &value); +// Atomic add for F32 values to global memory +// Uses ET hardware's custom amoaddg.w instruction for global atomic add +// This ensures correct accumulation when multiple threads contribute to the same output +static inline void atomic_add_f32(volatile float* addr, float value) { + uint32_t value_bits = *(uint32_t*)&value; + __asm__ volatile( + "amoaddg.w zero, %1, (%0)" + : + : "r"(addr), "r"(value_bits) + : "memory" + ); } -static inline void atomic_store_f16(volatile uint16_t * addr, uint16_t value) { - et_global_store_hw(addr, value); +// static inline void atomic_add_f32(volatile float* addr, float value) { +// // We use the "f" constraint to ensure 'value' is in a floating-point register (fs1) +// // and "r" for the address in an integer register (rs2) +// __asm__ volatile( +// "famoaddg.pi zero, %1, (%0)" +// : +// : "r"(addr), "f"(value) +// : "memory" +// ); +// } + +// // Atomic Floating Point Swap/Store Global +// static inline void atomic_store_f32(volatile float* addr, float value) { +// uint32_t value_bits = *(uint32_t*)&value; +// __asm__ volatile( +// "famoswapg.pi zero, %1, (%0)" +// : +// : "r"(addr), "f"(value) +// : "memory" +// ); +// } + +// static inline void atomic_add_f32(float *addr, float value) { +// uint32_t old_bits; +// uint32_t expected; +// uint32_t desired; + +// do { +// // Load current value +// __asm__ volatile ( +// "lw %0, 0(%1)" +// : "=r"(old_bits) +// : "r"(addr) +// : "memory" +// ); + +// float old_f; +// __builtin_memcpy(&old_f, &old_bits, sizeof(old_f)); +// float new_f = old_f + value; +// __builtin_memcpy(&desired, &new_f, sizeof(desired)); + +// expected = old_bits; + +// // CAS: rd, expected, desired, (addr) +// __asm__ volatile ( +// "amocmpswapg.w %0, %1, %2, (%3)" +// : "=r"(old_bits) +// : "r"(expected), "r"(desired), "r"(addr) +// : "memory" +// ); + +// } while (old_bits != expected); +// } + + +// static inline void atomic_store_f32(float *addr, float value) { +// uint32_t bits; +// __builtin_memcpy(&bits, &value, sizeof(bits)); + +// __asm__ volatile ( +// "amoswapg.w zero, %1, (%0)" +// : +// : "r"(addr), "r"(bits) +// : "memory" +// ); +// } + + +// Atomic store for F16 values to global memory +// Uses ET hardware's custom shg instruction (store halfword global) +// This ensures cache coherency when multiple threads write to nearby addresses +// Address must be 16-bit aligned +static inline void atomic_store_f16(volatile uint16_t* addr, uint16_t value) { + __asm__ volatile( + "shg %1, (%0)" + : + : "r"(addr), "r"(value) + : "memory" + ); } //****************************************************************************** @@ -542,4 +632,27 @@ static void evict_region_past_l2(const void * addr, size_t bytes) { } } +//****************************************************************************** +// Counter signaling between harts via L2 scratchpad (SCP) +//****************************************************************************** + +// Signal a counter value to the other hart via L2 SCP. +static inline void __attribute__((always_inline)) +scp_signal(volatile uint32_t *flag, uint32_t value) { + *flag = value; + FENCE; + evict_to_l2((const void *)flag, 1, 64); + WAIT_CACHEOPS; +} + +// Wait for a counter in L2 SCP to reach the expected value. +static inline void __attribute__((always_inline)) +scp_wait(volatile uint32_t *flag, uint32_t expected) { + while (1) { + evict_to_l2((const void *)flag, 1, 64); + WAIT_CACHEOPS; + if (*flag >= expected) return; + } +} + #endif // PLATFORM_H diff --git a/ggml/src/ggml-et/et-kernels/src/quants.h b/ggml/src/ggml-et/et-kernels/src/quants.h index 692ca00defe8..bb33a1e6c32e 100644 --- a/ggml/src/ggml-et/et-kernels/src/quants.h +++ b/ggml/src/ggml-et/et-kernels/src/quants.h @@ -33,6 +33,69 @@ static inline void dequantize_q4_0_block(const block_q4_0 * block, float * dst) } } +// Unpack the 12-byte packed Q3_K block scales into 16 signed 6-bit values. +static inline void unpack_q3_K_scales(const uint8_t * packed, int8_t * out /* [16] */) { + const uint32_t kmask1 = 0x03030303; + const uint32_t kmask2 = 0x0f0f0f0f; + uint32_t aux[4]; + __builtin_memcpy(aux, packed, 12); + const uint32_t tmp = aux[2]; + aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4); + aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4); + aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4); + aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4); + __builtin_memcpy(out, aux, 16); +} + +// Dequantize one Q2_K super-block (256 elements) to F32. +static inline void dequantize_q2_K_block(const block_q2_K * block, float * dst) { + const float d = fp16_to_fp32(block->d); + const float min = fp16_to_fp32(block->dmin); + const uint8_t * q = block->qs; + + int is = 0; + for (int n = 0; n < QK_K; n += 128) { + int shift = 0; + for (int j = 0; j < 4; ++j) { + uint8_t sc = block->scales[is++]; + float dl = d * (sc & 0xF), ml = min * (sc >> 4); + for (int l = 0; l < 16; ++l) *dst++ = dl * ((int8_t)((q[l] >> shift) & 3)) - ml; + sc = block->scales[is++]; + dl = d * (sc & 0xF); ml = min * (sc >> 4); + for (int l = 0; l < 16; ++l) *dst++ = dl * ((int8_t)((q[l + 16] >> shift) & 3)) - ml; + shift += 2; + } + q += 32; + } +} + +// Dequantize one Q3_K super-block (256 elements) to F32. +static inline void dequantize_q3_K_block(const block_q3_K * block, float * dst) { + const float d_all = fp16_to_fp32(block->d); + const uint8_t * q = block->qs; + const uint8_t * hm = block->hmask; + uint8_t m = 1; + + int8_t scales[16]; + unpack_q3_K_scales(block->scales, scales); + + int is = 0; + for (int n = 0; n < QK_K; n += 128) { + int shift = 0; + for (int j = 0; j < 4; ++j) { + float dl = d_all * (scales[is++] - 32); + for (int l = 0; l < 16; ++l) + *dst++ = dl * ((int8_t)((q[l + 0] >> shift) & 3) - ((hm[l + 0] & m) ? 0 : 4)); + dl = d_all * (scales[is++] - 32); + for (int l = 0; l < 16; ++l) + *dst++ = dl * ((int8_t)((q[l + 16] >> shift) & 3) - ((hm[l + 16] & m) ? 0 : 4)); + shift += 2; + m <<= 1; + } + q += 32; + } +} + // Unpack the 6-bit scale/min pair for Q4_K group j (groups 4-7 split their high bits). static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { if (j < 4) { @@ -69,4 +132,54 @@ static inline void dequantize_q4_K_block(const block_q4_K * block, float * dst) } } +// Dequantize one Q5_K super-block (256 elements) to F32. Same affine form as +// Q4_K with an extra high bit per weight drawn from qh. +static inline void dequantize_q5_K_block(const block_q5_K * block, float * dst) { + const uint8_t * ql = block->qs; + const uint8_t * qh = block->qh; + const float d = fp16_to_fp32(block->d); + const float min = fp16_to_fp32(block->dmin); + + int is = 0; + uint8_t sc, m; + uint8_t u1 = 1, u2 = 2; + for (int j = 0; j < QK_K; j += 64) { + get_scale_min_k4(is + 0, block->scales, &sc, &m); + const float d1 = d * sc, m1 = min * m; + get_scale_min_k4(is + 1, block->scales, &sc, &m); + const float d2 = d * sc, m2 = min * m; + for (int l = 0; l < 32; ++l) *dst++ = d1 * ((ql[l] & 0xF) + (qh[l] & u1 ? 16 : 0)) - m1; + for (int l = 0; l < 32; ++l) *dst++ = d2 * ((ql[l] >> 4) + (qh[l] & u2 ? 16 : 0)) - m2; + ql += 32; is += 2; + u1 <<= 2; u2 <<= 2; + } +} + +// Dequantize one Q6_K super-block (256 elements) to F32. Each 6-bit weight is +// (ql nibble | qh 2-bit) - 32, scaled by an int8 per-16 scale and the fp16 d. +static inline void dequantize_q6_K_block(const block_q6_K * block, float * dst) { + const float d = fp16_to_fp32(block->d); + const uint8_t * ql = block->ql; + const uint8_t * qh = block->qh; + const int8_t * sc = block->scales; + + for (int n = 0; n < QK_K; n += 128) { + for (int l = 0; l < 32; ++l) { + const int is = l / 16; + const int8_t q1 = (int8_t)((ql[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32; + const int8_t q2 = (int8_t)((ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32; + const int8_t q3 = (int8_t)((ql[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32; + const int8_t q4 = (int8_t)((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32; + dst[l + 0] = d * sc[is + 0] * q1; + dst[l + 32] = d * sc[is + 2] * q2; + dst[l + 64] = d * sc[is + 4] * q3; + dst[l + 96] = d * sc[is + 6] * q4; + } + dst += 128; + ql += 64; + qh += 32; + sc += 8; + } +} + #endif // QUANTS_H diff --git a/ggml/src/ggml-et/et-kernels/src/uberkernel.c b/ggml/src/ggml-et/et-kernels/src/uberkernel.c index 40d1cf9daa9b..af6c77cd8119 100644 --- a/ggml/src/ggml-et/et-kernels/src/uberkernel.c +++ b/ggml/src/ggml-et/et-kernels/src/uberkernel.c @@ -81,6 +81,11 @@ extern int mul_mat_f32_entry(struct ggml_et_binary_params *, void *); extern int mul_mat_f32_matrix_engine_entry(struct ggml_et_binary_params *, void *); extern int mul_mat_Q8_0_entry(struct ggml_et_mm_q8_params *, void *); extern int mul_mat_Q4_0_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_Q4_K_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_Q2_K_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_Q3_K_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_Q5_K_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_Q6_K_entry(struct ggml_et_binary_params *, void *); static inline size_t tensor_bytes(const struct ggml_tensor * t) { return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; @@ -483,6 +488,36 @@ int entry_point(struct ggml_et_uberkernel_params * params, void * env) { rc = mul_mat_Q4_0_entry(p, env); break; } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_Q4_K: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_Q4_K_entry(p, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_Q2_K: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_Q2_K_entry(p, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_Q3_K: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_Q3_K_entry(p, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_Q5_K: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_Q5_K_entry(p, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_Q6_K: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_Q6_K_entry(p, env); + break; + } default: return -1; diff --git a/ggml/src/ggml-et/ggml-et-cpu-compare.cpp b/ggml/src/ggml-et/ggml-et-cpu-compare.cpp index b37f6d261d97..12ed94262a31 100644 --- a/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +++ b/ggml/src/ggml-et/ggml-et-cpu-compare.cpp @@ -286,9 +286,6 @@ bool ggml_et_cpu_compare_compute_and_check(ggml_et_cpu_compare_ctx * ct } } break; - case GGML_OP_GET_ROWS: - ctx->cpu_dst = ggml_get_rows(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1); - break; case GGML_OP_CONT: ctx->cpu_dst = ggml_cont(ctx->ggml_ctx, ctx->cpu_src0); break; @@ -311,6 +308,9 @@ bool ggml_et_cpu_compare_compute_and_check(ggml_et_cpu_compare_ctx * ct ctx->cpu_dst = ggml_set_rows(ctx->ggml_ctx, cpu_dst_base, ctx->cpu_src0, ctx->cpu_src1); } break; + case GGML_OP_GET_ROWS: + ctx->cpu_dst = ggml_get_rows(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1); + break; default: GGML_LOG_ERROR("ET: Unsupported operation %s for CPU comparison\n", ggml_op_name(op)); return false; diff --git a/ggml/src/ggml-et/ggml-et-kernels.cpp b/ggml/src/ggml-et/ggml-et-kernels.cpp index 3e119283e082..fdda362b4dd6 100644 --- a/ggml/src/ggml-et/ggml-et-kernels.cpp +++ b/ggml/src/ggml-et/ggml-et-kernels.cpp @@ -96,6 +96,10 @@ static bool ggml_et_uberkernel_ensure_slot_capacity(ggml_backend_et_uberkernel_s return slot.device_insts != nullptr && slot.device_params != nullptr; } +#define ET_TRACE_DECODER_IMPL +#include +#include + // Get embedded kernel data by name static std::vector ggml_et_get_embedded_kernel(const std::string & kernel_name) { auto it = ggml_et_embedded_kernels.find(kernel_name); diff --git a/ggml/src/ggml-et/ggml-et-ops.cpp b/ggml/src/ggml-et/ggml-et-ops.cpp index 6c80fe8acde3..013cb59bc2d8 100644 --- a/ggml/src/ggml-et/ggml-et-ops.cpp +++ b/ggml/src/ggml-et/ggml-et-ops.cpp @@ -730,7 +730,59 @@ bool ggml_et_op_mul_mat(ggml_backend_et_device_context * dev_ctx, node->src[1]->type == GGML_TYPE_F32) { kernel_name = "mul_mat_Q4_0"; // N < 53, or M % 16 != 0 or K % 32 != 0 src0_type_name = "Q4_0"; - + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q4_K && + node->src[1]->type == GGML_TYPE_F32 && + node->src[1]->ne[1] >= 53 && // N >= 53 (prefill): use matrix engine + node->src[0]->ne[1] % 16 == 0 && // M % TILE_M + node->src[0]->ne[0] % 256 == 0) { // K % QK_K (Q4_K super-block) + kernel_name = "mul_mat_Q4_K_matrix_engine"; + src0_type_name = "Q4_K"; + } else if (node->type == GGML_TYPE_F32 && + node->src[0]->type == GGML_TYPE_Q4_K && + node->src[1]->type == GGML_TYPE_F32) { + + kernel_name = "mul_mat_Q4_K"; // N < 53, or M % 16 != 0, or K % 256 != 0 + src0_type_name = "Q4_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q6_K && + node->src[1]->type == GGML_TYPE_F32 && + node->src[1]->ne[1] >= 53 && // N >= 53 (prefill): use matrix engine + node->src[0]->ne[1] % 16 == 0 && // M % TILE_M + node->src[0]->ne[0] % 256 == 0) { // K % QK_K (Q6_K super-block) + kernel_name = "mul_mat_Q6_K_matrix_engine"; + src0_type_name = "Q6_K"; + } else if (node->type == GGML_TYPE_F32 && + node->src[0]->type == GGML_TYPE_Q6_K && + node->src[1]->type == GGML_TYPE_F32) { + + kernel_name = "mul_mat_Q6_K"; // N < 53, or M % 16 != 0, or K % 256 != 0 + src0_type_name = "Q6_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q2_K && + node->src[1]->type == GGML_TYPE_F32 && + node->src[1]->ne[1] >= 53 && node->src[0]->ne[1] % 16 == 0 && node->src[0]->ne[0] % 256 == 0) { + kernel_name = "mul_mat_Q2_K_matrix_engine"; + src0_type_name = "Q2_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q2_K && + node->src[1]->type == GGML_TYPE_F32) { + kernel_name = "mul_mat_Q2_K"; // N < 53, or M % 16 != 0, or K % 256 != 0 + src0_type_name = "Q2_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q3_K && + node->src[1]->type == GGML_TYPE_F32 && + node->src[1]->ne[1] >= 53 && node->src[0]->ne[1] % 16 == 0 && node->src[0]->ne[0] % 256 == 0) { + kernel_name = "mul_mat_Q3_K_matrix_engine"; + src0_type_name = "Q3_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q3_K && + node->src[1]->type == GGML_TYPE_F32) { + kernel_name = "mul_mat_Q3_K"; // N < 53, or M % 16 != 0, or K % 256 != 0 + src0_type_name = "Q3_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q5_K && + node->src[1]->type == GGML_TYPE_F32 && + node->src[1]->ne[1] >= 53 && node->src[0]->ne[1] % 16 == 0 && node->src[0]->ne[0] % 256 == 0) { + kernel_name = "mul_mat_Q5_K_matrix_engine"; + src0_type_name = "Q5_K"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q5_K && + node->src[1]->type == GGML_TYPE_F32) { + kernel_name = "mul_mat_Q5_K"; // N < 53, or M % 16 != 0, or K % 256 != 0 + src0_type_name = "Q5_K"; } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q8_0 && node->src[1]->type == GGML_TYPE_F32) { kernel_name = "mul_mat_Q8_0"; @@ -1621,7 +1673,9 @@ bool ggml_et_op_get_rows(ggml_backend_et_device_context * dev_ctx, const ggml_te if (node->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_I32 && (node->src[0]->type == GGML_TYPE_F32 || node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_Q4_0 || node->src[0]->type == GGML_TYPE_Q8_0 || - node->src[0]->type == GGML_TYPE_Q4_K)) { + node->src[0]->type == GGML_TYPE_Q4_K || node->src[0]->type == GGML_TYPE_Q6_K || + node->src[0]->type == GGML_TYPE_Q2_K || node->src[0]->type == GGML_TYPE_Q3_K || + node->src[0]->type == GGML_TYPE_Q5_K)) { kernel_name = "get_rows_f32"; } else { diff --git a/ggml/src/ggml-et/ggml-et.cpp b/ggml/src/ggml-et/ggml-et.cpp index b30209095672..6efb10a6a83f 100644 --- a/ggml/src/ggml-et/ggml-et.cpp +++ b/ggml/src/ggml-et/ggml-et.cpp @@ -717,6 +717,15 @@ static ggml_status ggml_backend_et_graph_compute(ggml_backend_t backend, ggml_cg case GGML_OP_MUL_MAT: ggml_et_op_mul_mat(dev_ctx, node); + + // if (once < 100){ + // uint64_t * host_data = (uint64_t *) node->data; + + // // printf("Tensor error: %lu\n", host_data[0]); + + // // printf("Tensor error:"); + // once++; + // } break; case GGML_OP_MUL_MAT_ID: @@ -1007,6 +1016,47 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm supported = src0_first_dim_contiguous && src1_first_dim_contiguous && dst_first_dim_contiguous && dst_properly_ordered; + } else if (op->type == GGML_TYPE_F32 && + op->src[0] && op->src[0]->type == GGML_TYPE_Q4_K && + op->src[1] && op->src[1]->type == GGML_TYPE_F32) { + + // Keep the existing quantized path constraints separate from the + // relaxed non-quant generic fallback. + bool src0_first_dim_contiguous = (op->src[0]->nb[0] == ggml_type_size(op->src[0]->type)); + bool src1_first_dim_contiguous = (op->src[1]->nb[0] == ggml_type_size(op->src[1]->type)); + bool dst_first_dim_contiguous = (op->nb[0] == sizeof(float)); + + bool dst_properly_ordered = true; + for (int d = 0; d < 3; d++) { + if (op->ne[d] > 1 && op->ne[d+1] > 1 && op->nb[d] > op->nb[d+1]) { + dst_properly_ordered = false; + } + } + + supported = src0_first_dim_contiguous && + src1_first_dim_contiguous && + dst_first_dim_contiguous && + dst_properly_ordered; + } else if (op->type == GGML_TYPE_F32 && + op->src[0] && (op->src[0]->type == GGML_TYPE_Q6_K || op->src[0]->type == GGML_TYPE_Q2_K || + op->src[0]->type == GGML_TYPE_Q3_K || op->src[0]->type == GGML_TYPE_Q5_K) && + op->src[1] && op->src[1]->type == GGML_TYPE_F32) { + + bool src0_first_dim_contiguous = (op->src[0]->nb[0] == ggml_type_size(op->src[0]->type)); + bool src1_first_dim_contiguous = (op->src[1]->nb[0] == ggml_type_size(op->src[1]->type)); + bool dst_first_dim_contiguous = (op->nb[0] == sizeof(float)); + + bool dst_properly_ordered = true; + for (int d = 0; d < 3; d++) { + if (op->ne[d] > 1 && op->ne[d+1] > 1 && op->nb[d] > op->nb[d+1]) { + dst_properly_ordered = false; + } + } + + supported = src0_first_dim_contiguous && + src1_first_dim_contiguous && + dst_first_dim_contiguous && + dst_properly_ordered; } else { supported = false; } @@ -1309,11 +1359,13 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm } break; case GGML_OP_GET_ROWS: - // Support F32/F16/Q4_0/Q8_0/Q4_K data with I32 indices -> F32 output + // Support F32/F16/Q4_0/Q8_0/Q4_K/Q6_K data with I32 indices -> F32 output if (op->type == GGML_TYPE_F32 && op->src[0] && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 || - op->src[0]->type == GGML_TYPE_Q4_K) && + op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q6_K || + op->src[0]->type == GGML_TYPE_Q2_K || op->src[0]->type == GGML_TYPE_Q3_K || + op->src[0]->type == GGML_TYPE_Q5_K) && op->src[1] && op->src[1]->type == GGML_TYPE_I32 && ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) { // Validate dimension constraints from ggml implementation diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 084344fb25d7..8435e65da54c 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8696,6 +8696,15 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 1, 3, 2})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 3, 2, 1})); + // [ET] Q4_K MUL_MAT at real Llama-3.2-1B shapes: large K (->use_ksplit path) and + // prefill batch N>=53. These exercise paths the K=256/N<=16 cases above never hit. + for (int64_t n : {1, 64}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 2048, n, 2048, {1, 1}, {1, 1})); // wq/wo + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 512, n, 2048, {1, 1}, {1, 1})); // wk/wv + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 2048, n, 8192, {1, 1}, {1, 1})); // ffn_down + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 8192, n, 2048, {1, 1}, {1, 1})); // ffn_gate/up (simple path, control) + } + for (ggml_type type_a : other_types) { for (ggml_type type_b : {GGML_TYPE_F32}) { if (ggml_blck_size(type_a) != 256) {