diff --git a/CMakeLists.txt b/CMakeLists.txt index e915f69d..855b5ae1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,4 +48,8 @@ else() message(STATUS "DLINFER_BUILD_DICP=OFF: skipping DICP vendor build (set env DLINFER_BUILD_DICP=ON to enable)") endif() +if(DEVICE STREQUAL "ascend") + add_subdirectory(dlinfer/vendor/ascend/csrc/grouped_matmul_direct) +endif() + install(CODE "message(STATUS \"Install completed for device: ${DEVICE}\")") diff --git a/benchmark/probe_grouped_matmul_direct.py b/benchmark/probe_grouped_matmul_direct.py new file mode 100644 index 00000000..2e5fea4b --- /dev/null +++ b/benchmark/probe_grouped_matmul_direct.py @@ -0,0 +1,71 @@ +"""Device correctness probe for DLInfer's bundled direct2560 extension.""" + +import argparse + +from dlinfer.vendor.ascend import grouped_matmul_direct +import torch +import torch_npu + + +def run(dtype: torch.dtype, iterations: int = 1) -> None: + if not grouped_matmul_direct.is_available(): + raise RuntimeError(grouped_matmul_direct.unavailable_reason()) + + experts, hidden_size, output_size, tokens = 2560, 16, 16, 8 + device = torch.device("npu:0") + x = torch.randn(tokens, hidden_size, device=device, dtype=dtype) + stored_weight = torch.randn( + experts, output_size, hidden_size, device=device, dtype=dtype + ) + weight = stored_weight.transpose(1, 2) + group_list = torch.zeros(experts, device=device, dtype=torch.int64) + group_list[:tokens] = 1 + + references = [] + row_start = 0 + for expert_start in range(0, experts, 1024): + expert_end = min(expert_start + 1024, experts) + chunk_groups = group_list[expert_start:expert_end] + row_end = row_start + int(chunk_groups.sum().cpu()) + if row_end > row_start: + references.append( + torch.ops.npu.npu_grouped_matmul( + [x[row_start:row_end]], + [weight[expert_start:expert_end]], + group_list=chunk_groups, + split_item=2, + group_type=0, + group_list_type=1, + )[0] + ) + row_start = row_end + + reference = torch.cat(references, dim=0) + for iteration in range(iterations): + print(f"ITERATION {iteration + 1}/{iterations} begin", flush=True) + direct = grouped_matmul_direct.grouped_matmul(x, stored_weight, group_list, 1) + print(f"ITERATION {iteration + 1}/{iterations} submitted", flush=True) + torch_npu.npu.synchronize() + print(f"ITERATION {iteration + 1}/{iterations} synchronized", flush=True) + torch.testing.assert_close(direct, reference, rtol=0, atol=0) + print( + f"PASS dtype={dtype} iterations={iterations} " + f"max_abs={(direct - reference).abs().max().item()}" + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--dtype", choices=("fp16", "bf16", "both"), default="both") + parser.add_argument("--iterations", type=int, default=1) + args = parser.parse_args() + if args.iterations < 1: + parser.error("--iterations must be at least 1") + if args.dtype in ("fp16", "both"): + run(torch.float16, args.iterations) + if args.dtype in ("bf16", "both"): + run(torch.bfloat16, args.iterations) + + +if __name__ == "__main__": + main() diff --git a/dlinfer/__init__.py b/dlinfer/__init__.py index e48da7cb..cedcbad9 100644 --- a/dlinfer/__init__.py +++ b/dlinfer/__init__.py @@ -1,4 +1,29 @@ # Copyright (c) 2024, DeepLink. All rights reserved. +import os +from pathlib import Path + + +def _register_bundled_ascend_opp() -> None: + """Expose wheel-local Ascend operators before torch_npu initializes.""" + if os.environ.get("DLINFER_GMM_EXPERIMENT", "chunked") != "direct2560": + return + ascend_dir = Path(__file__).parent / "vendor/ascend" + candidates = ( + ascend_dir / "grouped_matmul_direct", + ascend_dir / "csrc/grouped_matmul_direct/vendor", + ) + bundled_opp = next((path for path in candidates if path.is_dir()), None) + if bundled_opp is None: + return + bundled_opp_str = str(bundled_opp) + configured = os.environ.get("ASCEND_CUSTOM_OPP_PATH", "") + paths = [path for path in configured.split(":") if path] + if bundled_opp_str not in paths: + os.environ["ASCEND_CUSTOM_OPP_PATH"] = ":".join([bundled_opp_str, *paths]) + + +_register_bundled_ascend_opp() + import dlinfer.vendor as vendor vendor.vendor_torch_init() diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/CMakeLists.txt b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/CMakeLists.txt new file mode 100644 index 00000000..17d5128f --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/CMakeLists.txt @@ -0,0 +1,46 @@ +include(ascend) + +set(GMM_DIRECT_VENDOR_DIR ${CMAKE_CURRENT_SOURCE_DIR}/vendor) +set(GMM_DIRECT_OPAPI ${GMM_DIRECT_VENDOR_DIR}/op_api/lib/libcust_opapi.so) + +if(NOT EXISTS ${GMM_DIRECT_OPAPI}) + message(FATAL_ERROR "Missing bundled direct2560 op-api: ${GMM_DIRECT_OPAPI}") +endif() + +add_library(_grouped_matmul_direct MODULE grouped_matmul_direct.cpp) +find_library(TORCH_PYTHON_LIBRARY torch_python + PATHS "${TORCH_INSTALL_PREFIX}/lib" + REQUIRED + NO_DEFAULT_PATH +) +set_target_properties(_grouped_matmul_direct PROPERTIES + PREFIX "" + BUILD_RPATH "${GMM_DIRECT_VENDOR_DIR}/lib" + INSTALL_RPATH "\$ORIGIN/grouped_matmul_direct/op_api/lib" +) + +target_compile_definitions(_grouped_matmul_direct PRIVATE + TORCH_EXTENSION_NAME=_grouped_matmul_direct + _GLIBCXX_USE_CXX11_ABI=${_GLIBCXX_USE_CXX11_ABI} +) +target_compile_options(_grouped_matmul_direct PRIVATE -O2 -Wall -Wextra) +target_include_directories(_grouped_matmul_direct PRIVATE + ${GMM_DIRECT_VENDOR_DIR}/op_api/include + ${TORCH_INCLUDE_DIRS} + ${CANN_INCLUDE_DIRS} + ${CANN_INCLUDE_DIRS}/aclnn +) +target_link_libraries(_grouped_matmul_direct PRIVATE + Python::Python + torch + ${TORCH_PYTHON_LIBRARY} + ${GMM_DIRECT_OPAPI} + ${CANN_LIBRARIES} +) + +install(TARGETS _grouped_matmul_direct + LIBRARY DESTINATION dlinfer/vendor/ascend +) +install(DIRECTORY ${GMM_DIRECT_VENDOR_DIR}/ + DESTINATION dlinfer/vendor/ascend/grouped_matmul_direct +) diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/grouped_matmul_direct.cpp b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/grouped_matmul_direct.cpp new file mode 100644 index 00000000..b856fe9d --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/grouped_matmul_direct.cpp @@ -0,0 +1,163 @@ +// Copied from DeepLink-org/DLBlas csrc/ascend/grouped_matmul_direct and +// adapted for DLInfer pybind packaging. +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +constexpr int64_t kDirectGroups = 2560; + +at::Tensor GroupedMatmulDirect(const at::Tensor& x, const at::Tensor& weight, const at::Tensor& group_list, int64_t group_list_type, uint64_t stream_handle); + +aclDataType ToAclDataType(const at::ScalarType dtype) { + switch (dtype) { + case at::ScalarType::Half: + return ACL_FLOAT16; + case at::ScalarType::BFloat16: + return ACL_BF16; + case at::ScalarType::Long: + return ACL_INT64; + default: + TORCH_CHECK(false, "unsupported dtype for direct grouped matmul: ", dtype); + } +} + +struct AclTensorDeleter { + void operator()(aclTensor* tensor) const { + if (tensor != nullptr) { + aclDestroyTensor(tensor); + } + } +}; + +struct AclTensorListDeleter { + void operator()(aclTensorList* tensors) const { + if (tensors != nullptr) { + aclDestroyTensorList(tensors); + } + } +}; + +using AclTensorPtr = std::unique_ptr; +using AclTensorListPtr = std::unique_ptr; + +AclTensorPtr MakeAclTensor(const at::Tensor& tensor) { + std::vector shape(tensor.sizes().begin(), tensor.sizes().end()); + std::vector strides(tensor.strides().begin(), tensor.strides().end()); + auto* acl_tensor = aclCreateTensor(shape.data(), + shape.size(), + ToAclDataType(tensor.scalar_type()), + strides.data(), + tensor.storage_offset(), + ACL_FORMAT_ND, + shape.data(), + shape.size(), + tensor.data_ptr()); + TORCH_CHECK(acl_tensor != nullptr, "aclCreateTensor failed"); + return AclTensorPtr(acl_tensor); +} + +AclTensorPtr MakeTransposedWeightAclTensor(const at::Tensor& tensor) { + TORCH_CHECK(tensor.dim() == 3 && tensor.is_contiguous(), "stored weight must be contiguous [E, N, K]"); + const std::array view_shape{tensor.size(0), tensor.size(2), tensor.size(1)}; + const std::array view_strides{tensor.stride(0), tensor.stride(2), tensor.stride(1)}; + const std::array storage_shape{tensor.size(0), tensor.size(1), tensor.size(2)}; + auto* acl_tensor = aclCreateTensor(view_shape.data(), + view_shape.size(), + ToAclDataType(tensor.scalar_type()), + view_strides.data(), + tensor.storage_offset(), + ACL_FORMAT_ND, + storage_shape.data(), + storage_shape.size(), + tensor.data_ptr()); + TORCH_CHECK(acl_tensor != nullptr, "aclCreateTensor failed for weight"); + return AclTensorPtr(acl_tensor); +} + +AclTensorListPtr MakeAclTensorList(aclTensor* tensor) { + std::array tensors{tensor}; + auto* list = aclCreateTensorList(tensors.data(), tensors.size()); + TORCH_CHECK(list != nullptr, "aclCreateTensorList failed"); + return AclTensorListPtr(list); +} + +at::Tensor GroupedMatmulDirect(const at::Tensor& x, const at::Tensor& weight, const at::Tensor& group_list, int64_t group_list_type, uint64_t stream_handle) { + TORCH_CHECK(x.device().type() == c10::DeviceType::PrivateUse1, "x must be an NPU tensor"); + TORCH_CHECK(weight.device() == x.device() && group_list.device() == x.device(), "x, weight and group_list must be on the same NPU device"); + TORCH_CHECK(x.dim() == 2, "x must have shape [M, K]"); + TORCH_CHECK(weight.dim() == 3 && weight.size(0) == kDirectGroups, "weight must have stored shape [2560, N, K]"); + TORCH_CHECK(group_list.dim() == 1 && group_list.numel() == kDirectGroups, "group_list must contain 2560 entries"); + TORCH_CHECK(group_list.scalar_type() == at::ScalarType::Long, "group_list must be int64"); + TORCH_CHECK(group_list_type == 1, "direct2560 only supports group_list_type=1"); + TORCH_CHECK(x.scalar_type() == at::ScalarType::Half || x.scalar_type() == at::ScalarType::BFloat16, "direct2560 only supports float16 and bfloat16"); + TORCH_CHECK(weight.scalar_type() == x.scalar_type(), "x and weight dtypes must match"); + TORCH_CHECK(x.size(1) == weight.size(2), "x K must match weight K"); + + auto out = at::empty({x.size(0), weight.size(1)}, x.options()); + auto workspace = at::Tensor(); + + auto x_acl = MakeAclTensor(x); + auto weight_acl = MakeTransposedWeightAclTensor(weight); + auto group_list_acl = MakeAclTensor(group_list); + auto out_acl = MakeAclTensor(out); + auto x_list = MakeAclTensorList(x_acl.get()); + auto weight_list = MakeAclTensorList(weight_acl.get()); + auto out_list = MakeAclTensorList(out_acl.get()); + // aclTensorList owns the tensors passed to aclCreateTensorList. Transfer + // ownership to the lists so the individual guards do not destroy them a + // second time when this function returns. + x_acl.release(); + weight_acl.release(); + out_acl.release(); + + uint64_t workspace_size = 0; + aclOpExecutor* executor = nullptr; + const auto status = aclnnDlinferGroupedMatmulDirectV5GetWorkspaceSize(x_list.get(), + weight_list.get(), + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + group_list_acl.get(), + nullptr, + nullptr, + nullptr, + 2, + 0, + group_list_type, + 0, + nullptr, + out_list.get(), + nullptr, + nullptr, + &workspace_size, + &executor); + TORCH_CHECK(status == ACL_SUCCESS, "aclnnDlinferGroupedMatmulDirectV5GetWorkspaceSize failed, status=", status); + TORCH_CHECK(executor != nullptr, "GroupedMatmul returned a null executor"); + + void* workspace_addr = nullptr; + if (workspace_size > 0) { + workspace = at::empty({static_cast(workspace_size)}, x.options().dtype(at::ScalarType::Byte)); + workspace_addr = workspace.data_ptr(); + } + + TORCH_CHECK(stream_handle != 0, "current NPU stream handle must not be null"); + const auto stream = reinterpret_cast(stream_handle); + const auto execute_status = aclnnDlinferGroupedMatmulDirectV5(workspace_addr, workspace_size, executor, stream); + TORCH_CHECK(execute_status == ACL_SUCCESS, "aclnnDlinferGroupedMatmulDirectV5 failed, status=", execute_status); + return out; +} + +} // namespace + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, module) { module.def("grouped_matmul", &GroupedMatmulDirect, "DLInfer bundled GroupedMatmul direct2560"); } diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul.h b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul.h new file mode 100644 index 00000000..760a9af3 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul.h @@ -0,0 +1,63 @@ +/** + * Copyright (c) 2025 Huawei Technologies Co., Ltd. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + */ +#ifndef OP_API_INC_GROUPED_MATMUL_H +#define OP_API_INC_GROUPED_MATMUL_H +#include "aclnn/aclnn_base.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief aclnnDlinferGroupedMatmulDirect的第一段接口，根据具体的计算流程，计算workspace大小。 + * @domain aclnn_ops_infer + * + * @param [in] x: 表示公式中的x，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] weight: + * 表示公式中的weight，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] biasOptional: + * 表示公式中的bias，数据类型支持BLOAT16、FLOAT16、FLOAT32、INT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] scaleOptional: 表示量化参数，数据类型支持UINT64数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] offsetOptional: 表示量化参数，数据类型支持FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] antiquantScaleOptional: + * 表示伪量化参数，数据类型支持FLOAT16，BFLOAT16数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] antiquantOffsetOptional: + * 表示伪量化参数，数据类型支持FLOAT16，BFLOAT16数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] groupListOptional: 可选参数，代表输入和输出M轴上的索引情况，数据类型支持INT64，支持的最大长度为128个。 + * @param [in] splitItem: + * 整数型参数，代表输出是否要做tensor切分，0/1代表输出为多tensor；2/3代表输出为单tensor，默认值为0。 + * @param [out] y: 表示公式中的y，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [out] workspaceSize: 返回用户需要在npu device侧申请的workspace大小。 + * @param [out] executor: 返回op执行器，包含算子计算流程。 + * @return aclnnStatus: 返回状态码。 + */ +__attribute__((visibility("default"))) aclnnStatus aclnnDlinferGroupedMatmulDirectGetWorkspaceSize( + const aclTensorList* x, const aclTensorList* weight, const aclTensorList* biasOptional, + const aclTensorList* scaleOptional, const aclTensorList* offsetOptional, + const aclTensorList* antiquantScaleOptional, const aclTensorList* antiquantOffsetOptional, + const aclIntArray* groupListOptional, int64_t splitItem, const aclTensorList* y, uint64_t* workspaceSize, + aclOpExecutor** executor); + +/** + * @brief aclnnDlinferGroupedMatmulDirect的第二段接口，用于执行计算。 + * @param [in] workspace: 在npu device侧申请的workspace内存起址。 + * @param [in] workspaceSize: 在npu device侧申请的workspace大小，由第一段接口aclnnDlinferGroupedMatmulDirectGetWorkspaceSize获取。 + * @param [in] executor: op执行器，包含了算子计算流程。 + * @param [in] stream: acl stream流。 + * @return aclnnStatus: 返回状态码。 + */ +__attribute__((visibility("default"))) aclnnStatus aclnnDlinferGroupedMatmulDirect(void* workspace, uint64_t workspaceSize, aclOpExecutor* executor, + aclrtStream stream); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_910_95_checker.h b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_910_95_checker.h new file mode 100644 index 00000000..4b1c46ca --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_910_95_checker.h @@ -0,0 +1,83 @@ +/** + * Copyright (c) 2025 Huawei Technologies Co., Ltd. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + */ + +#ifndef OP_API_INC_GROUPED_MATMUL_910_95_CHECKER_H +#define OP_API_INC_GROUPED_MATMUL_910_95_CHECKER_H +#include "opdev/format_utils.h" +#include "aclnn_kernels/common/op_error_check.h" +#include "aclnn_grouped_matmul_util.h" + +namespace gmm { +template +class AclnnDlinferGroupedMatmulDirect91095Checker { +public: + explicit AclnnDlinferGroupedMatmulDirect91095Checker(const DlinferGroupedMatmulDirectParamsBase &gmmParams) : gmmParams_(gmmParams){}; + ~AclnnDlinferGroupedMatmulDirect91095Checker(){}; + aclnnStatus CheckDlinferGroupedMatmulDirect91095() const; + bool IsPerTileQuantMode() const; + void SetInputName(const std::string& xName, const std::string& weightName, const std::string& perTokenScaleName, + const std::string& scaleName, const std::string& groupTensorName); + +private: + struct TensorDimInfo { + size_t xDimNum = 0; + size_t weightDimNum = 0; + size_t scaleDimNum = 0; + size_t pertokenScaleDimNum = 0; + int64_t groupNum = 0; + size_t biasDimNum = 0; + }; + + bool IsQuant(DataType &xDtype, DataType &weightDtype) const; + aclnnStatus CheckGeneralQuantShape() const; + aclnnStatus CheckQuantCasesFormat() const; + + aclnnStatus CheckDlinferGroupedMatmulDirectMxDtype() const; + aclnnStatus CheckDlinferGroupedMatmulDirectPerGroupDim() const; + aclnnStatus CheckDlinferGroupedMatmulDirectMxShape() const; + aclnnStatus CheckDlinferGroupedMatmulDirectMxScaleTranspose() const; + aclnnStatus CheckDlinferGroupedMatmulDirectPerTile() const; + aclnnStatus CheckDlinferGroupedMatmulDirectPerTileShape() const; + aclnnStatus CheckDlinferGroupedMatmulDirectMxfp8() const; + aclnnStatus CheckDlinferGroupedMatmulDirectMxfp4() const; + aclnnStatus CheckDlinferGroupedMatmulDirectFp4MxDimValue() const; + + aclnnStatus CheckNonPerGroupQuantDim() const; + aclnnStatus CheckNonPerGroupQuantPertokenShape() const; + aclnnStatus CheckNonPerGroupQuantShape() const; + aclnnStatus CheckInt8QuantDtype() const; + aclnnStatus CheckInt8QuantParams() const; + aclnnStatus CheckFp8Hif8QuantParams() const; + aclnnStatus CheckFp8Params(const DataType &scaleDtype) const; + aclnnStatus CheckFp4Params(const DataType &scaleDtype) const; + aclnnStatus CheckNonMxQuantTransposeStatus() const; + bool CheckTensorListSizeForEachInput() const; + bool IsSpecialMXCase(const T *tensorList) const; + aclnnStatus CheckMxFp8TypeKCaseInputShape(const TensorDimInfo &dimInfo, size_t index) const; + aclnnStatus CheckMxTypeMCaseInputShape(const TensorDimInfo &dimInfo, size_t index) const; + aclnnStatus CheckMxBiasInputShape(const TensorDimInfo &dimInfo, size_t index) const; + bool LastTwoDimValueIsOne(const aclTensor *tensor) const; + bool IsSpecialperTileScene(int64_t groupNum, int64_t weightNDim, int64_t weightKDim, int64_t xMDim, + int64_t perTokenMDim) const; + +private: + DlinferGroupedMatmulDirectParamsBase gmmParams_; + std::string xName_ = "x"; + std::string weightName_ = "weight"; + std::string scaleName_ = "scale"; + std::string perTokenScaleName_ = "perTokenScale"; + std::string groupTensorName_ = "groupTensor"; + std::string biasName_ = "bias"; + std::string yName_ = "y"; + const std::vector SPECIAL_QUANT_DTYPES = {DataType::DT_FLOAT4_E1M2, DataType::DT_FLOAT4_E2M1, + DataType::DT_INT4}; +}; +} // namespace gmm +#endif \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_util.h b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_util.h new file mode 100644 index 00000000..aaa63e5f --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_util.h @@ -0,0 +1,136 @@ +/** + * Copyright (c) 2025 Huawei Technologies Co., Ltd. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + */ +#ifndef OP_API_INC_GROUPED_MATMUL_UTIL_H +#define OP_API_INC_GROUPED_MATMUL_UTIL_H +#include "opdev/common_types.h" +#include "opdev/op_executor.h" + +namespace gmm { +using namespace op; +constexpr int64_t NO_SPLIT = -1L; +constexpr int64_t SPLIT_M = 0L; +constexpr int64_t SPLIT_K = 2L; +constexpr int64_t SPLIT_N = 1L; + +constexpr int64_t GROUP_LIST_SPARSE_M = 2L; + +constexpr size_t MAX_FM_DIM = 6UL; +constexpr size_t MIN_FM_DIM = 2UL; +constexpr size_t SPLIT_M_SINGLE_WEIGHT_DIM = 3UL; +constexpr size_t SPLIT_K_SINGLE_WEIGHT_DIM = 2UL; +// mx dim num +constexpr size_t MX_SPLIT_M_SINGLE_X_DIM = 2UL; +constexpr size_t MX_SPLIT_M_SINGLE_WEIGHT_DIM = 3UL; +constexpr size_t MX_SPLIT_M_SCALE_DIM = 4UL; +constexpr size_t MX_BIAS_DIM = 2UL; +constexpr size_t MX_SPLIT_M_PER_TOKEN_SCALE_DIM = 3UL; +constexpr size_t MX_SPLIT_K_SINGLE_X_DIM = 2UL; +constexpr size_t MX_SPLIT_K_SINGLE_WEIGHT_DIM = 2UL; +constexpr size_t MX_SPLIT_K_SCALE_DIM = 3UL; +constexpr size_t MX_SPLIT_K_PER_TOKEN_SCALE_DIM = 3UL; + +constexpr size_t MX_TUPLE_X_DIM_INDEX = 0UL; +constexpr size_t MX_TUPLE_WEIGHT_DIM_INDEX = 1UL; +constexpr size_t MX_TUPLE_SCALE_DIM_INDEX = 2UL; +constexpr size_t MX_TUPLE_PERTOKENSCALE_DIM_INDEX = 3UL; +constexpr size_t MX_TUPLE_GROUP_NUMBER_DIM_INDEX = 4UL; + +constexpr int64_t PERTILE_GROUP_SIZE = 128L; +constexpr int64_t MXFP_DIVISOR_SIZE = 64L; +constexpr int64_t MXFP_MULTI_BASE_SIZE = 2L; +constexpr int64_t EVEN_FACTOR = 2L; +constexpr int64_t LAST_TWO_DIM_INDEX = 2L; + +constexpr int64_t N_K_MAX_VALUE_WEIGHT_QUANT = 65535L; +constexpr int64_t N_K_ALIGN_VALUE_WEIGHT_QUANT_4BIT = 64L; + +constexpr size_t LAST_FIRST_DIM_INDEX = 1; +constexpr size_t LAST_SECOND_DIM_INDEX = 2; +constexpr size_t LAST_THIRD_DIM_INDEX = 3; + +enum class GMMApiVersion : uint32_t { + V1 = 1U, + V2 = 2U, + V3 = 3U, + V4 = 4U, + V5 = 5U, + WeightNz = 6U +}; + +template +struct DlinferGroupedMatmulDirectParamsBase { + const T *x = nullptr; + const T *weight = nullptr; + const aclTensorList *biasOptional = nullptr; + const aclIntArray *groupListOptional = nullptr; + const aclTensor *groupTensorOptional = nullptr; + const T *scaleOptional = nullptr; + const aclTensorList *offsetOptional = nullptr; + const aclTensorList *antiquantScaleOptional = nullptr; + const aclTensorList *antiquantOffsetOptional = nullptr; + const T *perTokenScaleOptional = nullptr; + const aclTensorList *activationInputOptional = nullptr; + const aclTensorList *activationQuantScaleOptional = nullptr; + const aclTensorList *activationQuantOffsetOptional = nullptr; + int64_t splitItem = 0; + int64_t groupListType = 0; + int64_t activeType = 0; + bool transposeWeight = false; + bool transposeX = false; + bool isSingleWeight = false; + GMMApiVersion apiVersion = GMMApiVersion::V1; + int64_t groupType = -1L; + aclIntArray *tuningConfigOptional = nullptr; + const T *y = nullptr; + const aclTensorList *activationFeatureOutOptional = nullptr; + const aclTensorList *dynQuantScaleOutOptional = nullptr; + DataType xDtype = DataType::DT_FLOAT16; +}; + +using DlinferGroupedMatmulDirectParams = DlinferGroupedMatmulDirectParamsBase; + +const std::map BIAS_DTYPE { + {DataType::DT_FLOAT16, aclDataType::ACL_FLOAT16}, + {DataType::DT_BF16, aclDataType::ACL_FLOAT}, + {DataType::DT_INT8, aclDataType::ACL_INT32}, + {DataType::DT_FLOAT, aclDataType::ACL_FLOAT}, + {DataType::DT_FLOAT8_E4M3FN, aclDataType::ACL_FLOAT}, + {DataType::DT_FLOAT8_E5M2, aclDataType::ACL_FLOAT}, + {DataType::DT_HIFLOAT8, aclDataType::ACL_FLOAT}, + {DataType::DT_INT4, aclDataType::ACL_FLOAT16}, + {DataType::DT_FLOAT4_E1M2, aclDataType::ACL_FLOAT}, + {DataType::DT_FLOAT4_E2M1, aclDataType::ACL_FLOAT} +}; + +const std::map DTYPE_STRING{ + {DataType::DT_FLOAT16, "FLOAT16"}, + {DataType::DT_BF16, "BF16"}, + {DataType::DT_INT8, "INT8"}, + {DataType::DT_INT4, "INT4"}, + {DataType::DT_FLOAT, "FLOAT"}, + {DataType::DT_FLOAT8_E4M3FN, "FLOAT8_E4M3FN"}, + {DataType::DT_FLOAT8_E5M2, "FLOAT8_E5M2"}, + {DataType::DT_HIFLOAT8, "HIFLOAT8"} +}; + +const std::initializer_list DTYPE_SUPPORT_LIST = {DataType::DT_FLOAT, DataType::DT_FLOAT16, + DataType::DT_BF16}; + +bool IsTransposeLastTwoDims(const aclTensor *tensor); +bool IsTransposeForMxShape(const aclTensor *tensor); +void CreateContiguousTensorListForPertoken(const aclTensorList *tensorList, std::vector &newTensorList, + aclOpExecutor *executor); +void CreateContiguousTensorListForMXTypeMScale(const aclTensorList *tensorList, std::vector &newTensorList, + aclOpExecutor *executor); +void CreateContiguousTensorList(const aclTensorList *tensorList, std::vector &newTensorList, + aclOpExecutor *executor); +std::string dTypeToString(const ge::DataType &dtype); +} // namespace gmm +#endif \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_v2.h b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_v2.h new file mode 100644 index 00000000..9b9c2353 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_v2.h @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2025 Huawei Technologies Co., Ltd. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + */ +#ifndef OP_API_INC_GROUPED_MATMUL_V2_H +#define OP_API_INC_GROUPED_MATMUL_V2_H +#include "aclnn/aclnn_base.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief aclnnDlinferGroupedMatmulDirectV2的第一段接口，根据具体的计算流程，计算workspace大小。 + * @domain aclnn_ops_infer + * + * @param [in] x: 表示公式中的x，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] weight: + * 表示公式中的weight，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] biasOptional: + * 表示公式中的bias，数据类型支持BFLOAT16、FLOAT16、FLOAT32、INT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] scaleOptional: 表示量化参数，数据类型支持UINT64数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] offsetOptional: 表示量化参数，数据类型支持FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] antiquantScaleOptional: + * 表示伪量化参数，数据类型支持FLOAT16，BFLOAT16数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] antiquantOffsetOptional: + * 表示伪量化参数，数据类型支持FLOAT16，BFLOAT16数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] groupListOptional: 可选参数，代表输入和输出分组轴上的索引情况，数据类型支持INT64，支持的最大长度为128个。 + * @param [in] splitItem: + * 整数型参数，代表输出是否要做tensor切分，0/1代表输出为多tensor；2/3代表输出为单tensor，默认值为0。 + * @param [in] groupType: + * 整数型参数，代表需要切分的轴，-1代表不需要切分；0代表需要切分M轴；1代表需要切分N轴；2代表需要切分K轴。 + * @param [out] y: 表示公式中的out，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [out] workspaceSize: 返回用户需要在npu device侧申请的workspace大小。 + * @param [out] executor: 返回op执行器，包含算子计算流程。 + * @return aclnnStatus: 返回状态码。 + */ +__attribute__((visibility("default"))) aclnnStatus aclnnDlinferGroupedMatmulDirectV2GetWorkspaceSize( + const aclTensorList* x, const aclTensorList* weight, const aclTensorList* biasOptional, + const aclTensorList* scaleOptional, const aclTensorList* offsetOptional, + const aclTensorList* antiquantScaleOptional, const aclTensorList* antiquantOffsetOptional, + const aclIntArray* groupListOptional, int64_t splitItem, int64_t groupType, const aclTensorList* y, + uint64_t* workspaceSize, aclOpExecutor** executor); + +/** + * @brief aclnnDlinferGroupedMatmulDirectV2的第二段接口，用于执行计算。 + * @param [in] workspace: 在npu device侧申请的workspace内存起址。 + * @param [in] workspaceSize: 在npu device侧申请的workspace大小，由第一段接口aclnnDlinferGroupedMatmulDirectV2GetWorkspaceSize获取。 + * @param [in] executor: op执行器，包含了算子计算流程。 + * @param [in] stream: acl stream流。 + * @return aclnnStatus: 返回状态码。 + */ +__attribute__((visibility("default"))) aclnnStatus aclnnDlinferGroupedMatmulDirectV2(void* workspace, uint64_t workspaceSize, aclOpExecutor* executor, + aclrtStream stream); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_v3.h b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_v3.h new file mode 100644 index 00000000..83649953 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_v3.h @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2025 Huawei Technologies Co., Ltd. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + */ +#ifndef OP_API_INC_GROUPED_MATMUL_V3_H +#define OP_API_INC_GROUPED_MATMUL_V3_H +#include "aclnn/aclnn_base.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief aclnnDlinferGroupedMatmulDirectV3的第一段接口，根据具体的计算流程，计算workspace大小。 + * @domain aclnn_ops_infer + * + * @param [in] x: 表示公式中的x，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] weight: + * 表示公式中的weight，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] biasOptional: + * 表示公式中的bias，数据类型支持BFLOAT16、FLOAT16、FLOAT32、INT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] scaleOptional: 表示量化参数，数据类型支持UINT64数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] offsetOptional: 表示量化参数，数据类型支持FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] antiquantScaleOptional: + * 表示伪量化参数，数据类型支持FLOAT16，BFLOAT16数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] antiquantOffsetOptional: + * 表示伪量化参数，数据类型支持FLOAT16，BFLOAT16数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] groupListOptional: 可选参数，代表输入和输出分组轴上的索引情况，数据类型支持INT64，支持的最大长度为128个。 + * @param [in] splitItem: + * 整数型参数，代表输出是否要做tensor切分，0/1代表输出为多tensor；2/3代表输出为单tensor，默认值为0。 + * @param [in] groupType: + * 整数型参数，代表需要切分的轴，-1代表不需要切分；0代表需要切分M轴；1代表需要切分N轴；2代表需要切分K轴。 + * @param [out] y: 表示公式中的out，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [out] workspaceSize: 返回用户需要在npu device侧申请的workspace大小。 + * @param [out] executor: 返回op执行器，包含算子计算流程。 + * @return aclnnStatus: 返回状态码。 + */ +__attribute__((visibility("default"))) aclnnStatus aclnnDlinferGroupedMatmulDirectV3GetWorkspaceSize( + const aclTensorList* x, const aclTensorList* weight, const aclTensorList* biasOptional, + const aclTensorList* scaleOptional, const aclTensorList* offsetOptional, + const aclTensorList* antiquantScaleOptional, const aclTensorList* antiquantOffsetOptional, + const aclTensor* groupListOptional, int64_t splitItem, int64_t groupType, const aclTensorList* y, + uint64_t* workspaceSize, aclOpExecutor** executor); + +/** + * @brief aclnnDlinferGroupedMatmulDirectV3的第二段接口，用于执行计算。 + * @param [in] workspace: 在npu device侧申请的workspace内存起址。 + * @param [in] workspaceSize: 在npu device侧申请的workspace大小，由第一段接口aclnnDlinferGroupedMatmulDirectV3GetWorkspaceSize获取。 + * @param [in] executor: op执行器，包含了算子计算流程。 + * @param [in] stream: acl stream流。 + * @return aclnnStatus: 返回状态码。 + */ +__attribute__((visibility("default"))) aclnnStatus aclnnDlinferGroupedMatmulDirectV3(void* workspace, uint64_t workspaceSize, aclOpExecutor* executor, + aclrtStream stream); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_v4.h b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_v4.h new file mode 100644 index 00000000..f23dfa6f --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_v4.h @@ -0,0 +1,92 @@ +/** + * Copyright (c) 2025 Huawei Technologies Co., Ltd. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + */ +#ifndef OP_API_INC_GROUPED_MATMUL_V4_H +#define OP_API_INC_GROUPED_MATMUL_V4_H +#include "aclnn/aclnn_base.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + GMM_ACT_TYPE_NONE = 0L, + GMM_ACT_TYPE_RELU = 1L, + GMM_ACT_TYPE_GELU_TANH = 2L, + GMM_ACT_TYPE_GELU_ERR_FUNC = 3L, + GMM_ACT_TYPE_FAST_GELU = 4L, + GMM_ACT_TYPE_SILU = 5L, +} GMMActType; + +/** + * @brief aclnnDlinferGroupedMatmulDirectV4的第一段接口，根据具体的计算流程，计算workspace大小。 + * @domain aclnn_ops_infer + * + * @param [in] x: + * 表示公式中的x，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32、FLOAT8_E4M3FN、FLOAT8_E5M2、HIFLOAT8、FLOAT4_E1M2、FLOAT4_E2M1数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] weight: + * 表示公式中的weight，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32、INT4、FLOAT8_E4M3FN、FLOAT8_E5M2、HIFLOAT8、FLOAT4_E1M2、FLOAT4_E2M1数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] biasOptional: + * 表示公式中的bias，数据类型支持FLOAT16、FLOAT32、INT32、BFLOAT16数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] scaleOptional: + * 表示量化参数，数据类型支持UINT64、INT64、BFLOAT16、FLOAT32、FLOAT8_E8M0数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] offsetOptional: 表示量化参数，数据类型支持FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] antiquantScaleOptional: + * 表示伪量化参数，数据类型支持FLOAT16，BFLOAT16数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] antiquantOffsetOptional: + * 表示伪量化参数，数据类型支持FLOAT16，BFLOAT16数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [in] perTokenScaleOptional: + * 表示per token量化参数，数据类型支持FLOAT32、FLOAT8_E8M0数据类型，数据格式支持ND，支持的最大长度和x的长度一致。 + * @param [in] groupListOptional: 可选参数，代表输入和输出分组轴上的索引情况，数据类型支持INT64， + * 部分场景支持的最大长度为1024个（详见接口文档约束说明），其余场景支持的最大长度为128个。 + * @param [in] activationInputOptional: 可选参数，代表激活函数的反向输入。 + * @param [in] activationQuantScaleOptional: 可选参数，预留参数。 + * @param [in] activationQuantOffsetOptional: 可选参数，预留参数。 + * @param [in] splitItem: + * 整数型参数，代表输出是否要做tensor切分，0/1代表输出为多tensor；2/3代表输出为单tensor，默认值为0。 + * @param [in] groupType: + * 整数型参数，代表需要切分的轴，-1代表不需要切分；0代表需要切分M轴；1代表需要切分N轴；2代表需要切分K轴。 + * @param [in] groupListType: + * 整数型参数，可取值0或1，0代表groupListOptional中数值为分组轴大小的cumsum结果（累积和）， + * 1代表groupListOptional中数值为分组轴上每组大小。 + * @param [in] actType:整数型参数，代表激活函数类型，各激活函数枚举值参考枚举类GMMActType。 + * @param [out] out: 表示公式中的out，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [out] activationFeatureOutOptional: 激活函数的输入数据。 + * @param [out] dynQuantScaleOutOptional: 预留参数。 + * @param [out] workspaceSize: 返回用户需要在npu device侧申请的workspace大小。 + * @param [out] executor: 返回op执行器，包含算子计算流程。 + * @return aclnnStatus: 返回状态码。 + */ +__attribute__((visibility("default"))) aclnnStatus aclnnDlinferGroupedMatmulDirectV4GetWorkspaceSize( + const aclTensorList *x, const aclTensorList *weight, const aclTensorList *biasOptional, + const aclTensorList *scaleOptional, const aclTensorList *offsetOptional, + const aclTensorList *antiquantScaleOptional, const aclTensorList *antiquantOffsetOptional, + const aclTensorList *perTokenScaleOptional, const aclTensor *groupListOptional, + const aclTensorList *activationInputOptional, const aclTensorList *activationQuantScaleOptional, + const aclTensorList *activationQuantOffsetOptional, int64_t splitItem, int64_t groupType, + int64_t groupListType, int64_t actType, aclTensorList *out, aclTensorList *activationFeatureOutOptional, + aclTensorList *dynQuantScaleOutOptional, uint64_t *workspaceSize, + aclOpExecutor **executor); + +/** + * @brief aclnnDlinferGroupedMatmulDirectV4的第二段接口，用于执行计算。 + * @param [in] workspace: 在npu device侧申请的workspace内存起址。 + * @param [in] workspaceSize: 在npu device侧申请的workspace大小，由第一段接口aclnnDlinferGroupedMatmulDirectV4GetWorkspaceSize获取。 + * @param [in] executor: op执行器，包含了算子计算流程。 + * @param [in] stream: acl stream流。 + * @return aclnnStatus: 返回状态码。 + */ +__attribute__((visibility("default"))) aclnnStatus aclnnDlinferGroupedMatmulDirectV4(void* workspace, uint64_t workspaceSize, aclOpExecutor* executor, + aclrtStream stream); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_v5.h b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_v5.h new file mode 100644 index 00000000..7830b9fa --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_v5.h @@ -0,0 +1,75 @@ +/** + * Copyright (c) 2025 Huawei Technologies Co., Ltd. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + */ +#ifndef OP_API_INC_GROUPED_MATMUL_V5_H +#define OP_API_INC_GROUPED_MATMUL_V5_H +#include "aclnn/aclnn_base.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief aclnnDlinferGroupedMatmulDirectV5的第一段接口，根据具体的计算流程，计算workspace大小。 + * @domain aclnn_ops_infer + * + * @param [in] x: 表示公式中的输入x，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32，数据格式支持ND，支持的最大长度为128个。 + * @param [in] weight：表示公式中的weight，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32、FLOAT8_E4M3FN、HIFLOAT8、FLOAT8_E5M2，数据格式支持ND，支持的最大长度为128个。 + * @param [in] biasOptional：表示公式中的bias，数据类型支持BFLOAT16、FLOAT16、FLOAT32、INT32，数据格式支持ND，长度与weight相同。 + * @param [in] scaleOptional：代表量化参数中的缩放因子，数据类型支持UINT64，数据格式支持ND，长度与weight相同。 + * @param [in] offsetOptional：代表量化参数中的偏移量，数据类型支持FLOAT32，数据格式支持ND，长度与weight相同。 + * @param [in] antiquantScaleOptional：代表伪量化参数中的缩放因子，数据类型支持FLOAT16、BFLOAT16，数据格式支持ND，长度与weight相同。 + * @param [in] antiquantOffsetOptional：代表伪量化参数中的偏移量，数据类型支持FLOAT16、BFLOAT16，数据格式支持ND，长度与weight相同。 + * @param [in] perTokenScaleOptional：代表量化参数中的由输入x量化引入的缩放因子，数据类型支持FLOAT32、BFLOAT16，数据格式支持ND，长度与x相同 + * @param [in] groupListOptional：代表输入和输出分组轴的matmul大小分布，数据类型支持INT64，数据格式支持ND，长度与weight相同。 + * @param [in] activationInputOptional：可选参数，激活函数的反向输入。 + * @param [in] activationQuantScaleOptional：可选参数，激活函数的输出的量化系数。 + * @param [in] activationQuantOffsetOptional：可选参数，激活函数的输出的量化偏移。 + * @param [in] splitItem：整数型参数，代表输出是否要做tensor切分，0/1代表输出为多tensor；2/3代表输出为单tensor，默认值为0。 + * @param [in] groupType： + * 整数型参数，代表需要分组的轴，如矩阵乘为C[m,n]=A[m,k]xB[k,n]，则groupType取值-1：不分组，0：m轴分组，1：n轴分组，2：k轴分组，默认值为-1，当前不支持n轴和k轴分组。 + * @param [in] groupListType： + * 整数型参数，可取值0或1，0代表groupListOptional中数值为分组轴大小的cumsum结果（累积和），1代表groupListOptional中数值为各个分组轴大小，默认值为0。 + * @param [in] actType：整数型参数，代表激活函数类型，当前只支持0，无激活函数。 + * @param [in] tuningConfigOptional： + * 调优参数。数组中第一个值表示各个专家处理的token数的预期值，算子tiling时会按照该预期值进行最优tiling。 + * @param [out] out: 表示公式中的out，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [out] activationFeatureOutOptional: 激活函数的输入数据。 + * @param [out] dynQuantScaleOutOptional: 存在激活函数的时候，对激活后的输出进行动态量化的量化系数输出。 + * @param [out] workspaceSize: 返回用户需要在npu device侧申请的workspace大小。 + * @param [out] executor: 返回op执行器，包含了算子计算流程 + * @return aclnnStatus: 返回状态码。 + */ +__attribute__((visibility("default"))) aclnnStatus aclnnDlinferGroupedMatmulDirectV5GetWorkspaceSize( + const aclTensorList *x, const aclTensorList *weight, const aclTensorList *biasOptional, + const aclTensorList *scaleOptional, const aclTensorList *offsetOptional, + const aclTensorList *antiquantScaleOptional, const aclTensorList *antiquantOffsetOptional, + const aclTensorList *perTokenScaleOptional, const aclTensor *groupListOptional, + const aclTensorList *activationInputOptional, const aclTensorList *activationQuantScaleOptional, + const aclTensorList *activationQuantOffsetOptional, int64_t splitItem, int64_t groupType, + int64_t groupListType, int64_t actType, aclIntArray *tuningConfigOptional, aclTensorList *out, aclTensorList *activationFeatureOutOptional, + aclTensorList *dynQuantScaleOutOptional, uint64_t *workspaceSize, + aclOpExecutor **executor); + +/** + * @brief aclnnDlinferGroupedMatmulDirectV5的第二段接口，用于执行计算。 + * @param [in] workspace: 在npu device侧申请的workspace内存起址。 + * @param [in] workspaceSize: 在npu device侧申请的workspace大小，由第一段接口aclnnDlinferGroupedMatmulDirectV5GetWorkspaceSize获取。 + * @param [in] executor: op执行器，包含了算子计算流程。 + * @param [in] stream: acl stream流。 + * @return aclnnStatus: 返回状态码。 + */ +__attribute__((visibility("default"))) aclnnStatus aclnnDlinferGroupedMatmulDirectV5(void* workspace, uint64_t workspaceSize, aclOpExecutor* executor, + aclrtStream stream); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_weight_nz.h b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_weight_nz.h new file mode 100644 index 00000000..682eb35f --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_weight_nz.h @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2025 Huawei Technologies Co., Ltd. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + */ +#ifndef OP_API_INC_GROUPED_MATMUL_WEIGHT_NZ_H +#define OP_API_INC_GROUPED_MATMUL_WEIGHT_NZ_H +#include "aclnn/aclnn_base.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief aclnnDlinferGroupedMatmulDirectWeightNz的第一段接口，根据具体的计算流程，计算workspace大小。 + * @domain aclnn_ops_infer + * + * @param [in] x: 表示公式中的输入x，数据类型支持FLOAT16、BFLOAT16、INT8、FLOAT8_E4M3FN，数据格式支持ND，支持的最大长度为128个。 + * @param [in] weight：表示公式中的weight，数据类型支持FLOAT16、BFLOAT16、INT8、INT4、FLOAT4_E2M1、INT32、FLOAT32，数据格式支持NZ，支持的最大长度为128个。 + * @param [in] biasOptional：表示公式中的bias，数据类型支持FLOAT16、BFLOAT16、FLOAT32、INT32，数据格式支持ND，长度与weight相同。 + * @param [in] scaleOptional：代表量化参数中的缩放因子，数据类型支持BFLOAT16、FLOAT32、UINT64，数据格式支持ND，长度与weight相同。 + * @param [in] offsetOptional：代表量化参数中的偏移量，数据类型支持FLOAT32，数据格式支持ND，长度与weight相同。 + * @param [in] antiquantScaleOptional：代表伪量化参数中的缩放因子，数据类型支持FLOAT16、BFLOAT16、FLOAT8_E8M0，数据格式支持ND，长度与weight相同。 + * @param [in] antiquantOffsetOptional：代表伪量化参数中的偏移量，数据类型支持FLOAT16、BFLOAT16，数据格式支持ND，长度与weight相同。 + * @param [in] perTokenScaleOptional：代表量化参数中的由输入x量化引入的缩放因子，数据类型支持FLOAT32、FLOAT8_E8M0，数据格式支持ND，长度与x相同 + * @param [in] groupListOptional：代表输入和输出分组轴的matmul大小分布，数据类型支持INT64，数据格式支持ND，长度与weight相同。 + * @param [in] activationInputOptional：可选参数，激活函数的反向输入。 + * @param [in] activationQuantScaleOptional：可选参数，激活函数的输出的量化系数。 + * @param [in] activationQuantOffsetOptional：可选参数，激活函数的输出的量化偏移。 + * @param [in] splitItem：整数型参数，代表输出是否要做tensor切分，0/1代表输出为多tensor；2/3代表输出为单tensor，默认值为0。 + * @param [in] groupType： + * 整数型参数，代表需要分组的轴，如矩阵乘为C[m,n]=A[m,k]xB[k,n]，则groupType取值-1：不分组，0：m轴分组，1：n轴分组，2：k轴分组，默认值为-1，当前不支持n轴和k轴分组。 + * @param [in] groupListType： + * 整数型参数，可取值0或1，0代表groupListOptional中数值为分组轴大小的cumsum结果（累积和），1代表groupListOptional中数值为各个分组轴大小，默认值为0。 + * @param [in] actType：整数型参数，代表激活函数类型，当前只支持0，无激活函数。 + * @param [in] tuningConfigOptional： + * 调优参数。数组中第一个值表示各个专家处理的token数的预期值，算子tiling时会按照该预期值进行最优tiling。 + * @param [in] quantGroupSize: 整数型参数，代表分组量化（per-group）的分组大小，不涉及分组量化时，填0。 + * @param [out] out: 表示公式中的out，数据类型支持FLOAT16、BFLOAT16、INT32数据类型，数据格式支持ND，支持的最大长度为128个。 + * @param [out] activationFeatureOutOptional: 激活函数的输入数据。 + * @param [out] dynQuantScaleOutOptional: 存在激活函数的时候，对激活后的输出进行动态量化的量化系数输出。 + * @param [out] workspaceSize: 返回用户需要在npu device侧申请的workspace大小。 + * @param [out] executor: 返回op执行器，包含了算子计算流程 + * @return aclnnStatus: 返回状态码。 + */ +__attribute__((visibility("default"))) aclnnStatus aclnnDlinferGroupedMatmulDirectWeightNzGetWorkspaceSize( + const aclTensorList *x, const aclTensorList *weight, const aclTensorList *biasOptional, + const aclTensorList *scaleOptional, const aclTensorList *offsetOptional, + const aclTensorList *antiquantScaleOptional, const aclTensorList *antiquantOffsetOptional, + const aclTensorList *perTokenScaleOptional, const aclTensor *groupListOptional, + const aclTensorList *activationInputOptional, const aclTensorList *activationQuantScaleOptional, + const aclTensorList *activationQuantOffsetOptional, int64_t splitItem, int64_t groupType, + int64_t groupListType, int64_t actType, aclIntArray *tuningConfigOptional, int64_t quantGroupSize, aclTensorList *out, aclTensorList *activationFeatureOutOptional, + aclTensorList *dynQuantScaleOutOptional, uint64_t *workspaceSize, + aclOpExecutor **executor); + +/** + * @brief aclnnDlinferGroupedMatmulDirectWeightNz的第二段接口，用于执行计算。 + * @param [in] workspace: 在npu device侧申请的workspace内存起址。 + * @param [in] workspaceSize: 在npu device侧申请的workspace大小，由第一段接口aclnnDlinferGroupedMatmulDirectWeightNzGetWorkspaceSize获取。 + * @param [in] stream: acl stream流。 + * @param [in] executor: op执行器，包含了算子计算流程。 + * @return aclnnStatus: 返回状态码。 + */ +__attribute__((visibility("default"))) aclnnStatus aclnnDlinferGroupedMatmulDirectWeightNz(void* workspace, uint64_t workspaceSize, aclOpExecutor* executor, + aclrtStream stream); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_weight_quant_910_95_checker.h b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_weight_quant_910_95_checker.h new file mode 100644 index 00000000..cfd6f872 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_grouped_matmul_weight_quant_910_95_checker.h @@ -0,0 +1,66 @@ +/** + * Copyright (c) 2025 Huawei Technologies Co., Ltd. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + */ + +#ifndef OP_API_INC_GROUPED_MATMUL_WEIGHT_QUANT_910_95_CHECKER_H +#define OP_API_INC_GROUPED_MATMUL_WEIGHT_QUANT_910_95_CHECKER_H +#include "opdev/format_utils.h" +#include "aclnn_kernels/common/op_error_check.h" +#include "aclnn_grouped_matmul_util.h" + +namespace gmm { +class AclnnDlinferGroupedMatmulDirectWeightQuant91095Checker { +public: + explicit AclnnDlinferGroupedMatmulDirectWeightQuant91095Checker(const DlinferGroupedMatmulDirectParams &gmmParams) : gmmParams_(gmmParams){}; + ~AclnnDlinferGroupedMatmulDirectWeightQuant91095Checker(){}; + aclnnStatus CheckDlinferGroupedMatmulDirectWeightQuant91095(); + +private: + aclnnStatus CheckGroupTypeScenario() const; + aclnnStatus CheckUnsupportedApi() const; + aclnnStatus CheckGroupListAndSplitItem() const; + aclnnStatus CheckAntiQuantParams() const; + aclnnStatus CheckQuantParams() const; + aclnnStatus CheckYDtype() const; + aclnnStatus CheckQuantDtype() const; + aclnnStatus CheckBiasDtype(); + + aclnnStatus CheckTensorListSize() const; + aclnnStatus CheckTensorNotNull(size_t idx) const; + aclnnStatus CheckTensorNotNullPtr(const aclTensorList *tensorList, size_t idx, const std::string &tensorType) const; + aclnnStatus CheckTensorDtype(const aclTensorList *tensorList, const DataType &tensorDtype, size_t idx, + const std::string &tensorType) const; + aclnnStatus CheckTensorShape(const aclTensorList *tensorList, size_t idx, const std::string &tensorType) const; + + aclnnStatus CheckWeightInnerAxisEven(size_t idx) const; + aclnnStatus CheckDimNumAndFormat(size_t idx) const; + aclnnStatus CheckTransposeStatus() const; + aclnnStatus CheckDimValue(size_t idx) const; + aclnnStatus CheckV1GroupList(size_t idx) const; + + aclnnStatus CheckAntiQuantDtype(size_t idx) const; + aclnnStatus CheckScaleAndPerTokenScaleShape() const; + aclnnStatus CheckGroupSize(size_t idx) const; + + bool IsA16MxFp4NZ() const; + bool IsMxA8W4NZ() const; + bool IsA16W8ND() const; + bool IsA16F8ND() const; + bool IsS8S4NZ() const; + bool IsA16W4() const; + +private: + DlinferGroupedMatmulDirectParams gmmParams_; + DataType xDtype_ = ge::DT_UNDEFINED; + DataType weightDtype_ = ge::DT_UNDEFINED; + DataType biasDtype_ = ge::DT_UNDEFINED; + DataType yDtype_ = ge::DT_UNDEFINED; +}; +} // namespace gmm +#endif \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_ops_transformer_dlinfer_gmm_direct.h b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_ops_transformer_dlinfer_gmm_direct.h new file mode 100644 index 00000000..7878375b --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/include/aclnnop/aclnn_ops_transformer_dlinfer_gmm_direct.h @@ -0,0 +1,24 @@ +/** + * Copyright (c) 2025 Huawei Technologies Co., Ltd. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + */ + +#ifndef ACLNN_OPS_TRANSFORMER_DLINFER_GMM_DIRECT_H_ +#define ACLNN_OPS_TRANSFORMER_DLINFER_GMM_DIRECT_H_ + +#include "aclnn_grouped_matmul.h" +#include "aclnn_grouped_matmul_910_95_checker.h" +#include "aclnn_grouped_matmul_util.h" +#include "aclnn_grouped_matmul_v2.h" +#include "aclnn_grouped_matmul_v3.h" +#include "aclnn_grouped_matmul_v4.h" +#include "aclnn_grouped_matmul_v5.h" +#include "aclnn_grouped_matmul_weight_nz.h" +#include "aclnn_grouped_matmul_weight_quant_910_95_checker.h" + +#endif // ACLNN_OPS_TRANSFORMER_DLINFER_GMM_DIRECT_H_ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/lib/libcust_opapi.so b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/lib/libcust_opapi.so new file mode 100755 index 00000000..c85d6bbc Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_api/lib/libcust_opapi.so differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/config/ascend910_93/aic-ascend910_93-ops-info.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/config/ascend910_93/aic-ascend910_93-ops-info.json new file mode 100644 index 00000000..064761a3 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/config/ascend910_93/aic-ascend910_93-ops-info.json @@ -0,0 +1,158 @@ +{ + "DlinferGroupedMatmulDirect":{ + "aclnnSupport":{ + "value":"support_aclnn" + }, + "attr":{ + "list":"split_item,dtype,transpose_weight,transpose_x,group_type,group_list_type,act_type,tuning_config" + }, + "attr_act_type":{ + "defaultValue":"0", + "paramType":"optional", + "type":"int", + "value":"all" + }, + "attr_dtype":{ + "defaultValue":"0", + "paramType":"optional", + "type":"int", + "value":"all" + }, + "attr_group_list_type":{ + "defaultValue":"0", + "paramType":"optional", + "type":"int", + "value":"all" + }, + "attr_group_type":{ + "defaultValue":"-1", + "paramType":"optional", + "type":"int", + "value":"all" + }, + "attr_split_item":{ + "defaultValue":"0", + "paramType":"optional", + "type":"int", + "value":"all" + }, + "attr_transpose_weight":{ + "defaultValue":"false", + "paramType":"optional", + "type":"bool", + "value":"all" + }, + "attr_transpose_x":{ + "defaultValue":"false", + "paramType":"optional", + "type":"bool", + "value":"all" + }, + "attr_tuning_config":{ + "defaultValue":"[0]", + "paramType":"optional", + "type":"listInt", + "value":"all" + }, + "coreType":{ + "value":"AiCore" + }, + "dynamicCompileStatic":{ + "flag":"true" + }, + "dynamicFormat":{ + "flag":"true" + }, + "dynamicRankSupport":{ + "flag":"true" + }, + "dynamicShapeSupport":{ + "flag":"true" + }, + "input0":{ + "dtype":"float16,bfloat16,int8,float16,bfloat16,float32,int8,int8,int8,int8,float16,bfloat16,float16,bfloat16,int8,int8,int8,int8,int8,int8,int4,int4,int4,int4,int8,int8", + "format":"ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND", + "name":"x", + "paramType":"dynamic", + "shape":"all" + }, + "input1":{ + "dtype":"float16,bfloat16,int8,int8,int8,float32,int8,int8,int8,int8,int4,int4,float16,bfloat16,int8,int8,int4,int4,int4,int4,int4,int4,int4,int4,int8,int8", + "format":"ND,ND,ND,ND,ND,ND,ND,FRACTAL_NZ,ND,FRACTAL_NZ,ND,ND,FRACTAL_NZ,FRACTAL_NZ,ND,FRACTAL_NZ,FRACTAL_NZ,FRACTAL_NZ,ND,ND,FRACTAL_NZ,FRACTAL_NZ,ND,ND,FRACTAL_NZ,FRACTAL_NZ", + "name":"weight", + "paramType":"dynamic", + "shape":"all" + }, + "input2":{ + "dtype":"float16,float32,int32,float16,float32,float32,int32,int32,int32,int32,float16,float32,float16,float32,int32,int32,float32,float32,float32,float32,float16,float16,float16,float16,bfloat16,bfloat16", + "format":"ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND", + "name":"bias", + "paramType":"dynamic", + "shape":"all" + }, + "input3":{ + "dtype":"uint64,uint64,uint64,uint64,uint64,uint64,bfloat16,bfloat16,float32,float32,uint64,uint64,uint64,uint64,uint64,uint64,uint64,uint64,uint64,uint64,uint64,uint64,uint64,uint64,bfloat16,float32", + "format":"ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND", + "name":"scale", + "paramType":"dynamic", + "shape":"all" + }, + "input4":{ + "dtype":"float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32", + "format":"ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND", + "name":"offset", + "paramType":"dynamic", + "shape":"all" + }, + "input5":{ + "dtype":"float16,float16,float16,float16,bfloat16,float16,float16,float16,float16,float16,float16,bfloat16,float16,float16,float16,float16,float16,float16,float16,float16,float16,float16,float16,float16,float16,float16", + "format":"ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND", + "name":"antiquant_scale", + "paramType":"dynamic", + "shape":"all" + }, + "input6":{ + "dtype":"float16,float16,float16,float16,bfloat16,float16,float16,float16,float16,float16,float16,bfloat16,float16,float16,float16,float16,float16,float16,float16,float16,float16,float16,float16,float16,float16,float16", + "format":"ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND", + "name":"antiquant_offset", + "paramType":"dynamic", + "shape":"all" + }, + "input7":{ + "dtype":"int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64,int64", + "format":"ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND", + "name":"group_list", + "paramType":"optional", + "shape":"all" + }, + "input8":{ + "dtype":"float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32,float32", + "format":"ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND", + "name":"per_token_scale", + "paramType":"optional", + "shape":"all" + }, + "needCheckSupport":{ + "flag":"false" + }, + "opFile":{ + "value":"dlinfer_grouped_matmul_direct" + }, + "opInterface":{ + "value":"dlinfer_grouped_matmul_direct" + }, + "output0":{ + "dtype":"float16,bfloat16,int8,float16,bfloat16,float32,bfloat16,bfloat16,float16,float16,float16,bfloat16,float16,bfloat16,int32,int32,bfloat16,float16,bfloat16,float16,float16,bfloat16,float16,bfloat16,bfloat16,bfloat16", + "format":"ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND,ND", + "name":"y", + "paramType":"dynamic", + "shape":"all" + }, + "prebuildPattern":{ + "value":"Opaque" + }, + "precision_reduce":{ + "flag":"true" + } + } +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c.json new file mode 100644 index 00000000..41c7d06d --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c.json @@ -0,0 +1,316 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "3b130ab44566fa95256632333676815823cbb8476941d0f5ff9b9c601fc0724e", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 197122, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c_197122" + }, + { + "tilingKey": 268632578, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c_268632578" + }, + { + "tilingKey": 67305986, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c_67305986" + }, + { + "tilingKey": 335741442, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c_335741442" + }, + { + "tilingKey": 134414850, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c_134414850" + }, + { + "tilingKey": 402850306, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c_402850306" + }, + { + "tilingKey": 33751554, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c_33751554" + }, + { + "tilingKey": 302187010, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c_302187010" + }, + { + "tilingKey": 100860418, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c_100860418" + }, + { + "tilingKey": 369295874, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c_369295874" + }, + { + "tilingKey": 167969282, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c_167969282" + }, + { + "tilingKey": 436404738, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c_436404738" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/3,2/10,2/0,2/1,2/1,2/3,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "8b41b4586c56d0b2acef879821e22a2d5d883c4ad53c549732dcdab2f8ca509c,fa6b402a3f5197c0f0cc15f13817b0399903a58934cf9b4eedfdd70862a9d147", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c.o new file mode 100644 index 00000000..a0c2363d Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31.json new file mode 100644 index 00000000..be302708 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31.json @@ -0,0 +1,260 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "2612bee39437ee5f85f3ebf0ab561a1fc2b25c9ac39f2fc2a21addfd7ab5750c", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 68721253661, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31_68721253661" + }, + { + "tilingKey": 68788362525, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31_68788362525" + }, + { + "tilingKey": 68719549725, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31_68719549725" + }, + { + "tilingKey": 68786658589, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31_68786658589" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/29,2/29,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/29,2/29,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/29,2/29,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/29,2/29,29/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "13629ba98b813b7dbab170a22eab0bf06e7285a915cbc129d237325e15b6ba66,d9bf188a31acbb54fdb5028b92225dcac4701b0917b44041c2995023017aa1ad", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31.o new file mode 100644 index 00000000..dc2f1672 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2.json new file mode 100644 index 00000000..b657fb71 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2.json @@ -0,0 +1,421 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "e757fe154eab23a2b0b8b0b9ef6de98e6f9ae6eb845bc4eb41009515240efad8", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 1776411, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_1776411" + }, + { + "tilingKey": 68885275, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_68885275" + }, + { + "tilingKey": 135994139, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_135994139" + }, + { + "tilingKey": 65793, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_65793" + }, + { + "tilingKey": 67174657, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_67174657" + }, + { + "tilingKey": 134283521, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_134283521" + }, + { + "tilingKey": 0, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_0" + }, + { + "tilingKey": 67108864, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_67108864" + }, + { + "tilingKey": 134217728, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_134217728" + }, + { + "tilingKey": 35330843, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_35330843" + }, + { + "tilingKey": 102439707, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_102439707" + }, + { + "tilingKey": 169548571, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_169548571" + }, + { + "tilingKey": 33620225, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_33620225" + }, + { + "tilingKey": 100729089, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_100729089" + }, + { + "tilingKey": 167837953, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_167837953" + }, + { + "tilingKey": 33554432, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_33554432" + }, + { + "tilingKey": 100663296, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_100663296" + }, + { + "tilingKey": 167772160, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_167772160" + }, + { + "tilingKey": 34378291995, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_34378291995" + }, + { + "tilingKey": 34445400859, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_34445400859" + }, + { + "tilingKey": 34512509723, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_34512509723" + }, + { + "tilingKey": 34376581377, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_34376581377" + }, + { + "tilingKey": 34443690241, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_34443690241" + }, + { + "tilingKey": 34510799105, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_34510799105" + }, + { + "tilingKey": 34376515584, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_34376515584" + }, + { + "tilingKey": 34443624448, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_34443624448" + }, + { + "tilingKey": 34510733312, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2_34510733312" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/27,2/27,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/27,2/27,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/27,2/27,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/27,2/27,29/0,2/10,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "d4e39095f87cf6a91932800684b59e22dbac70172f185f4b43e4ee2a4d4a5da5,0ddc3ebcab5f87244f7fb305c17262fdc6568a5358d99481c865f521f6e6e220", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "bfloat16", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2.o new file mode 100644 index 00000000..728bd77d Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b.json new file mode 100644 index 00000000..687b65cc --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b.json @@ -0,0 +1,316 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "557dcfdd134c64d929cfe297adccad1176843f1350e96d75b7cd5ae77e6c304e", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 68721253659, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b_68721253659" + }, + { + "tilingKey": 68788362523, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b_68788362523" + }, + { + "tilingKey": 68719549697, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b_68719549697" + }, + { + "tilingKey": 68786658561, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b_68786658561" + }, + { + "tilingKey": 34361515291, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b_34361515291" + }, + { + "tilingKey": 34428624155, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b_34428624155" + }, + { + "tilingKey": 34395069723, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b_34395069723" + }, + { + "tilingKey": 34462178587, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b_34462178587" + }, + { + "tilingKey": 34359811329, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b_34359811329" + }, + { + "tilingKey": 34426920193, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b_34426920193" + }, + { + "tilingKey": 34393365761, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b_34393365761" + }, + { + "tilingKey": 34460474625, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b_34460474625" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/1,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/1,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/1,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/1,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "b341252294cb8f5035344367a36a364c6b361e1b9d4fccc79dea908593dcba83,7b05f5a3db1225db3287298ef758c10399ec9d331ad4d64d2f77b9cf1f812a0b", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b.o new file mode 100644 index 00000000..960a6acb Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c.json new file mode 100644 index 00000000..7d7b4e48 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c.json @@ -0,0 +1,260 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "1b5caa84204b3b084390bf87782e86c2754f60a56a772c37d565e1bb8b8f5d36", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 68721253661, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c_68721253661" + }, + { + "tilingKey": 68788362525, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c_68788362525" + }, + { + "tilingKey": 68719549725, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c_68719549725" + }, + { + "tilingKey": 68786658589, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c_68786658589" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/29,2/29,29/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/29,2/29,29/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/29,2/29,29/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/29,2/29,29/1,2/10,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "18a1653b6a965a7b7b23b9969942498b985ccb4631b9090d517ea5878f304f5a,2740b74e9b27b6447f39836565aba3e76317bc29a3d4c4d7e0624990da64122f", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c.o new file mode 100644 index 00000000..45690c32 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a.json new file mode 100644 index 00000000..12714e11 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a.json @@ -0,0 +1,302 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "809bd503abe2290d85e16b447045935e82984ee6cabd3670d2420c7b8955d349", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 69325233410, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a_69325233410" + }, + { + "tilingKey": 69323529474, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a_69323529474" + }, + { + "tilingKey": 69862104322, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a_69862104322" + }, + { + "tilingKey": 69860400386, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a_69860400386" + }, + { + "tilingKey": 36039236866, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a_36039236866" + }, + { + "tilingKey": 36037532930, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a_36037532930" + }, + { + "tilingKey": 70935846146, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a_70935846146" + }, + { + "tilingKey": 70934142210, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a_70934142210" + }, + { + "tilingKey": 71472717058, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a_71472717058" + }, + { + "tilingKey": 71471013122, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a_71471013122" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/29,29/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/29,29/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/29,29/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/29,29/0,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "a4847c0b99929f7ae30bdcd5dcb5becaeba41561fc1b097028f9e21258768db7,f85725b8721a243ce333e83f9bd0647c8f072d1162577de2c9fd34dd6859c64d", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a.o new file mode 100644 index 00000000..5a659d90 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f.json new file mode 100644 index 00000000..a41305d8 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f.json @@ -0,0 +1,316 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "9f7ce613ae16243f5a298d8bceee265e7046172bb00e3919ff6c72060500d114", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 197122, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f_197122" + }, + { + "tilingKey": 268632578, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f_268632578" + }, + { + "tilingKey": 67305986, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f_67305986" + }, + { + "tilingKey": 335741442, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f_335741442" + }, + { + "tilingKey": 134414850, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f_134414850" + }, + { + "tilingKey": 402850306, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f_402850306" + }, + { + "tilingKey": 33751554, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f_33751554" + }, + { + "tilingKey": 302187010, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f_302187010" + }, + { + "tilingKey": 100860418, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f_100860418" + }, + { + "tilingKey": 369295874, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f_369295874" + }, + { + "tilingKey": 167969282, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f_167969282" + }, + { + "tilingKey": 436404738, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f_436404738" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,2/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,2/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,2/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,2/3,2/10,2/0,2/1,2/1,2/3,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "8059ba9619837992c932ff680fb253301eaa4c20ea03fc8c4972d8cf67c60687,22bbb045a142913adde66ef6b543c7e71945d61243cdd268fc2c10f14bfef71a", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f.o new file mode 100644 index 00000000..a9845ed5 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb.json new file mode 100644 index 00000000..1516373c --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb.json @@ -0,0 +1,463 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "2e4710e4537d13a0dc74e8913ae062b6a55b1685c4d4fa44c0854c514b54bb8f", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 34361508354, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34361508354" + }, + { + "tilingKey": 34629943810, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34629943810" + }, + { + "tilingKey": 34428617218, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34428617218" + }, + { + "tilingKey": 34697052674, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34697052674" + }, + { + "tilingKey": 34495726082, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34495726082" + }, + { + "tilingKey": 34764161538, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34764161538" + }, + { + "tilingKey": 34395062786, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34395062786" + }, + { + "tilingKey": 34663498242, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34663498242" + }, + { + "tilingKey": 34462171650, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34462171650" + }, + { + "tilingKey": 34730607106, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34730607106" + }, + { + "tilingKey": 34529280514, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34529280514" + }, + { + "tilingKey": 34797715970, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34797715970" + }, + { + "tilingKey": 34359804418, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34359804418" + }, + { + "tilingKey": 34628239874, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34628239874" + }, + { + "tilingKey": 34426913282, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34426913282" + }, + { + "tilingKey": 34695348738, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34695348738" + }, + { + "tilingKey": 34494022146, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34494022146" + }, + { + "tilingKey": 34762457602, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34762457602" + }, + { + "tilingKey": 34393358850, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34393358850" + }, + { + "tilingKey": 34661794306, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34661794306" + }, + { + "tilingKey": 34460467714, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34460467714" + }, + { + "tilingKey": 34728903170, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34728903170" + }, + { + "tilingKey": 34527576578, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34527576578" + }, + { + "tilingKey": 34796012034, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_34796012034" + }, + { + "tilingKey": 68721246722, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_68721246722" + }, + { + "tilingKey": 68788355586, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_68788355586" + }, + { + "tilingKey": 68754801154, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_68754801154" + }, + { + "tilingKey": 68821910018, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_68821910018" + }, + { + "tilingKey": 68719542786, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_68719542786" + }, + { + "tilingKey": 68786651650, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_68786651650" + }, + { + "tilingKey": 68753097218, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_68753097218" + }, + { + "tilingKey": 68820206082, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_68820206082" + }, + { + "tilingKey": 171798757890, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb_171798757890" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,2/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,2/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,2/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,2/3,2/0,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "45aad95117637f87eb605ae9ed2c2879a00fea3f4b749f77b4415ee545c91046,544a277ccb408894775bb088b1a9f718ba38223aafa0823f78ffa51a27e0653b", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb.o new file mode 100644 index 00000000..7b4152e5 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049.json new file mode 100644 index 00000000..2217aaf4 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049.json @@ -0,0 +1,316 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "04008fefbb8712383492beef6b57b6a292a216faf6682c872c8a5eee99038517", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 131586, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049_131586" + }, + { + "tilingKey": 268567042, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049_268567042" + }, + { + "tilingKey": 67240450, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049_67240450" + }, + { + "tilingKey": 335675906, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049_335675906" + }, + { + "tilingKey": 134349314, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049_134349314" + }, + { + "tilingKey": 402784770, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049_402784770" + }, + { + "tilingKey": 33686018, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049_33686018" + }, + { + "tilingKey": 302121474, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049_302121474" + }, + { + "tilingKey": 100794882, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049_100794882" + }, + { + "tilingKey": 369230338, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049_369230338" + }, + { + "tilingKey": 167903746, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049_167903746" + }, + { + "tilingKey": 436339202, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049_436339202" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,2/3,2/10,2/0,2/1,2/1,2/2,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,2/3,2/10,2/0,2/1,2/1,2/2,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,2/3,2/10,2/0,2/1,2/1,2/2,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,2/3,2/10,2/0,2/1,2/1,2/2,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "739a802e05a38a09071755a4cda95d56820a00674eb74107b484bb956c755be5,b7a249aa6a17cca837a75b0df16d3d1c7cd1fa1664365bb412254c2bc106645f", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049.o new file mode 100644 index 00000000..9b460563 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf.json new file mode 100644 index 00000000..54ccebe7 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf.json @@ -0,0 +1,456 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "2ae49fbd67bc8dd622e54c0e26cb67ea06a234f4b48921d78aea21dbdf57b5d8", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 68721253659, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_68721253659" + }, + { + "tilingKey": 68788362523, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_68788362523" + }, + { + "tilingKey": 68719549697, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_68719549697" + }, + { + "tilingKey": 68786658561, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_68786658561" + }, + { + "tilingKey": 34361515291, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_34361515291" + }, + { + "tilingKey": 34428624155, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_34428624155" + }, + { + "tilingKey": 34395069723, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_34395069723" + }, + { + "tilingKey": 34462178587, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_34462178587" + }, + { + "tilingKey": 34359811329, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_34359811329" + }, + { + "tilingKey": 34426920193, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_34426920193" + }, + { + "tilingKey": 34393365761, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_34393365761" + }, + { + "tilingKey": 34460474625, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_34460474625" + }, + { + "tilingKey": 42951442971, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_42951442971" + }, + { + "tilingKey": 51541377563, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_51541377563" + }, + { + "tilingKey": 43018551835, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_43018551835" + }, + { + "tilingKey": 51608486427, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_51608486427" + }, + { + "tilingKey": 42984997403, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_42984997403" + }, + { + "tilingKey": 51574931995, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_51574931995" + }, + { + "tilingKey": 43052106267, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_43052106267" + }, + { + "tilingKey": 51642040859, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_51642040859" + }, + { + "tilingKey": 42949739009, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_42949739009" + }, + { + "tilingKey": 51539673601, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_51539673601" + }, + { + "tilingKey": 43016847873, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_43016847873" + }, + { + "tilingKey": 51606782465, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_51606782465" + }, + { + "tilingKey": 42983293441, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_42983293441" + }, + { + "tilingKey": 51573228033, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_51573228033" + }, + { + "tilingKey": 43050402305, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_43050402305" + }, + { + "tilingKey": 51640336897, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_51640336897" + }, + { + "tilingKey": 85901115931, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_85901115931" + }, + { + "tilingKey": 85968224795, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_85968224795" + }, + { + "tilingKey": 85899411969, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_85899411969" + }, + { + "tilingKey": 85966520833, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf_85966520833" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/1,2/2,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/1,2/2,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/1,2/2,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/1,2/2,2/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "a2a29e208da73038001e15c9f5afba2f89b3394b48b702e5418de2635ed8084d,61a5fd1fd70cdb16b149aca1874ecf9d74f9c32a84b8ec3322edd432bf6c0e07", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf.o new file mode 100644 index 00000000..e89d52a1 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952.json new file mode 100644 index 00000000..6c65a484 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952.json @@ -0,0 +1,421 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "eba3fe7a1c7b560b3e6fc416630d0b315479c8f31ee952e7a0bb306b7446ebcf", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 1776411, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_1776411" + }, + { + "tilingKey": 68885275, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_68885275" + }, + { + "tilingKey": 135994139, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_135994139" + }, + { + "tilingKey": 65793, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_65793" + }, + { + "tilingKey": 67174657, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_67174657" + }, + { + "tilingKey": 134283521, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_134283521" + }, + { + "tilingKey": 0, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_0" + }, + { + "tilingKey": 67108864, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_67108864" + }, + { + "tilingKey": 134217728, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_134217728" + }, + { + "tilingKey": 35330843, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_35330843" + }, + { + "tilingKey": 102439707, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_102439707" + }, + { + "tilingKey": 169548571, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_169548571" + }, + { + "tilingKey": 33620225, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_33620225" + }, + { + "tilingKey": 100729089, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_100729089" + }, + { + "tilingKey": 167837953, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_167837953" + }, + { + "tilingKey": 33554432, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_33554432" + }, + { + "tilingKey": 100663296, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_100663296" + }, + { + "tilingKey": 167772160, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_167772160" + }, + { + "tilingKey": 34378291995, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_34378291995" + }, + { + "tilingKey": 34445400859, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_34445400859" + }, + { + "tilingKey": 34512509723, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_34512509723" + }, + { + "tilingKey": 34376581377, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_34376581377" + }, + { + "tilingKey": 34443690241, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_34443690241" + }, + { + "tilingKey": 34510799105, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_34510799105" + }, + { + "tilingKey": 34376515584, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_34376515584" + }, + { + "tilingKey": 34443624448, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_34443624448" + }, + { + "tilingKey": 34510733312, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952_34510733312" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/1,2/1,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/1,2/1,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/1,2/1,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/1,2/1,29/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "4fcb19cc155249cee35987efc2a997f60ebb50b7ad0c22120b1237bd975d58eb,5cf37210872c01ad8c8f05d6a8c5d1f42cfbab310986687eb859b1bd802d1e7c", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "float16", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952.o new file mode 100644 index 00000000..0057abd6 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0.json new file mode 100644 index 00000000..d2ac2755 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0.json @@ -0,0 +1,302 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "056c16dd68602d9fda8d05958d20adac3fc70e3ffe9a3e5252ba7d7aef5b5e4f", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 69325233410, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0_69325233410" + }, + { + "tilingKey": 69323529474, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0_69323529474" + }, + { + "tilingKey": 69862104322, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0_69862104322" + }, + { + "tilingKey": 69860400386, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0_69860400386" + }, + { + "tilingKey": 36039236866, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0_36039236866" + }, + { + "tilingKey": 36037532930, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0_36037532930" + }, + { + "tilingKey": 70935846146, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0_70935846146" + }, + { + "tilingKey": 70934142210, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0_70934142210" + }, + { + "tilingKey": 71472717058, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0_71472717058" + }, + { + "tilingKey": 71471013122, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0_71471013122" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/29,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/29,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/29,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/29,2/0,2/10,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "b78628594c90cbd20276f56387a3fe7d70992796683188dac9585f2f2c3843d0,eb6ba8895861666637b0bc4575c4dba66e669e558e2ff95fa9e721644e74a38d", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0.o new file mode 100644 index 00000000..aa8e2e5d Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a.json new file mode 100644 index 00000000..d59e0800 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a.json @@ -0,0 +1,463 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "b1d791a4a2280184dd0f346f367cf27ea8f19dce8106f579596722e8a636c919", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 34361508354, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34361508354" + }, + { + "tilingKey": 34629943810, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34629943810" + }, + { + "tilingKey": 34428617218, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34428617218" + }, + { + "tilingKey": 34697052674, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34697052674" + }, + { + "tilingKey": 34495726082, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34495726082" + }, + { + "tilingKey": 34764161538, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34764161538" + }, + { + "tilingKey": 34395062786, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34395062786" + }, + { + "tilingKey": 34663498242, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34663498242" + }, + { + "tilingKey": 34462171650, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34462171650" + }, + { + "tilingKey": 34730607106, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34730607106" + }, + { + "tilingKey": 34529280514, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34529280514" + }, + { + "tilingKey": 34797715970, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34797715970" + }, + { + "tilingKey": 34359804418, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34359804418" + }, + { + "tilingKey": 34628239874, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34628239874" + }, + { + "tilingKey": 34426913282, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34426913282" + }, + { + "tilingKey": 34695348738, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34695348738" + }, + { + "tilingKey": 34494022146, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34494022146" + }, + { + "tilingKey": 34762457602, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34762457602" + }, + { + "tilingKey": 34393358850, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34393358850" + }, + { + "tilingKey": 34661794306, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34661794306" + }, + { + "tilingKey": 34460467714, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34460467714" + }, + { + "tilingKey": 34728903170, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34728903170" + }, + { + "tilingKey": 34527576578, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34527576578" + }, + { + "tilingKey": 34796012034, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_34796012034" + }, + { + "tilingKey": 68721246722, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_68721246722" + }, + { + "tilingKey": 68788355586, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_68788355586" + }, + { + "tilingKey": 68754801154, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_68754801154" + }, + { + "tilingKey": 68821910018, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_68821910018" + }, + { + "tilingKey": 68719542786, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_68719542786" + }, + { + "tilingKey": 68786651650, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_68786651650" + }, + { + "tilingKey": 68753097218, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_68753097218" + }, + { + "tilingKey": 68820206082, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_68820206082" + }, + { + "tilingKey": 171798757890, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a_171798757890" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/3,2/27,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "7ee92b880724aef3a3277ca7649a512d291908a635012aa90e015adcaf741903,ac024fac9c65de08068669ae7ea632fcbc14b835feee61f458bde27efc192482", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a.o new file mode 100644 index 00000000..2b25b761 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8.json new file mode 100644 index 00000000..698068be --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8.json @@ -0,0 +1,421 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "c5c57cfedd9f2d07c06924f0041d387fc839bbb891a51d63a37e497b4e5df4be", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 1776411, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_1776411" + }, + { + "tilingKey": 68885275, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_68885275" + }, + { + "tilingKey": 135994139, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_135994139" + }, + { + "tilingKey": 65793, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_65793" + }, + { + "tilingKey": 67174657, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_67174657" + }, + { + "tilingKey": 134283521, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_134283521" + }, + { + "tilingKey": 0, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_0" + }, + { + "tilingKey": 67108864, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_67108864" + }, + { + "tilingKey": 134217728, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_134217728" + }, + { + "tilingKey": 35330843, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_35330843" + }, + { + "tilingKey": 102439707, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_102439707" + }, + { + "tilingKey": 169548571, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_169548571" + }, + { + "tilingKey": 33620225, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_33620225" + }, + { + "tilingKey": 100729089, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_100729089" + }, + { + "tilingKey": 167837953, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_167837953" + }, + { + "tilingKey": 33554432, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_33554432" + }, + { + "tilingKey": 100663296, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_100663296" + }, + { + "tilingKey": 167772160, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_167772160" + }, + { + "tilingKey": 34378291995, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_34378291995" + }, + { + "tilingKey": 34445400859, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_34445400859" + }, + { + "tilingKey": 34512509723, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_34512509723" + }, + { + "tilingKey": 34376581377, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_34376581377" + }, + { + "tilingKey": 34443690241, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_34443690241" + }, + { + "tilingKey": 34510799105, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_34510799105" + }, + { + "tilingKey": 34376515584, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_34376515584" + }, + { + "tilingKey": 34443624448, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_34443624448" + }, + { + "tilingKey": 34510733312, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8_34510733312" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/27,2/27,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/27,2/27,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/27,2/27,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/27,2/27,2/0,2/10,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "e3fe5fb04d6809c2a00838a1a4bfea5de45c4034c057e608ab6b277da815a07a,fc282608b185db5ba03378f8a51f56cf2e3c06d947dccce5551d62f055cca6dd", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8.o new file mode 100644 index 00000000..dfb76c93 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a.json new file mode 100644 index 00000000..04f8be8e --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a.json @@ -0,0 +1,260 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "863ef1a10a75da89d0a53ba822df6e4ed064c2dde1c587b20e01a8d8fd740657", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 68721253661, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a_68721253661" + }, + { + "tilingKey": 68788362525, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a_68788362525" + }, + { + "tilingKey": 68719549725, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a_68719549725" + }, + { + "tilingKey": 68786658589, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a_68786658589" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/29,2/29,2/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/29,2/29,2/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/29,2/29,2/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/29,2/29,2/1,2/10,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "1ed4764a2067b34de7ad9316df128d7c57f4b86c00310be6ad812b79ca1e56d4,369fb10680882b00990ee64c6a9168a7c35a891b48fa35c532a84505a8cd59a6", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a.o new file mode 100644 index 00000000..6d1b96a6 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2.json new file mode 100644 index 00000000..f9bad656 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2.json @@ -0,0 +1,302 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "858b9648ebb8c8afa40c03ef5b292ad258f70c3057df16725db177de31752fc0", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 69325233410, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2_69325233410" + }, + { + "tilingKey": 69323529474, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2_69323529474" + }, + { + "tilingKey": 69862104322, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2_69862104322" + }, + { + "tilingKey": 69860400386, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2_69860400386" + }, + { + "tilingKey": 36039236866, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2_36039236866" + }, + { + "tilingKey": 36037532930, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2_36037532930" + }, + { + "tilingKey": 70935846146, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2_70935846146" + }, + { + "tilingKey": 70934142210, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2_70934142210" + }, + { + "tilingKey": 71472717058, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2_71472717058" + }, + { + "tilingKey": 71471013122, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2_71471013122" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/29,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/29,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/29,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/29,29/0,2/10,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "48b2e2a4c56aac6e2ffa2314d3a7e4ff90095eb3952dfdda0e3ed89ac831d60a,1448128e62c4dfbd70471cdb63281ff760f4ded0ff3fd7a5671c6c75308ddc7b", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2.o new file mode 100644 index 00000000..32a9d572 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45.json new file mode 100644 index 00000000..bc45792c --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45.json @@ -0,0 +1,421 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "b69f3ca8423fee224f4f78568536fd85839cc396ddfa8238f41642d2f4cd5d48", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 1776411, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_1776411" + }, + { + "tilingKey": 68885275, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_68885275" + }, + { + "tilingKey": 135994139, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_135994139" + }, + { + "tilingKey": 65793, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_65793" + }, + { + "tilingKey": 67174657, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_67174657" + }, + { + "tilingKey": 134283521, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_134283521" + }, + { + "tilingKey": 0, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_0" + }, + { + "tilingKey": 67108864, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_67108864" + }, + { + "tilingKey": 134217728, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_134217728" + }, + { + "tilingKey": 35330843, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_35330843" + }, + { + "tilingKey": 102439707, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_102439707" + }, + { + "tilingKey": 169548571, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_169548571" + }, + { + "tilingKey": 33620225, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_33620225" + }, + { + "tilingKey": 100729089, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_100729089" + }, + { + "tilingKey": 167837953, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_167837953" + }, + { + "tilingKey": 33554432, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_33554432" + }, + { + "tilingKey": 100663296, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_100663296" + }, + { + "tilingKey": 167772160, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_167772160" + }, + { + "tilingKey": 34378291995, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_34378291995" + }, + { + "tilingKey": 34445400859, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_34445400859" + }, + { + "tilingKey": 34512509723, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_34512509723" + }, + { + "tilingKey": 34376581377, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_34376581377" + }, + { + "tilingKey": 34443690241, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_34443690241" + }, + { + "tilingKey": 34510799105, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_34510799105" + }, + { + "tilingKey": 34376515584, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_34376515584" + }, + { + "tilingKey": 34443624448, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_34443624448" + }, + { + "tilingKey": 34510733312, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45_34510733312" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/1,2/1,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/1,2/1,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/1,2/1,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/1,2/1,2/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "7642f687f619d1419cccee1f466a3624fb1be887b15029a41aab74acab2e247b,3a506a2e7d3168ad349cbf48232cdaa6d81af8783e4a3b179adc1c11a8249b62", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45.o new file mode 100644 index 00000000..27fb3dab Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603.json new file mode 100644 index 00000000..438c07c9 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603.json @@ -0,0 +1,302 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "c75c46837a63d2cf980d21d815a7f1e3698f9094377fa466df6a228fffe1c651", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 69325233410, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603_69325233410" + }, + { + "tilingKey": 69323529474, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603_69323529474" + }, + { + "tilingKey": 69862104322, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603_69862104322" + }, + { + "tilingKey": 69860400386, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603_69860400386" + }, + { + "tilingKey": 36039236866, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603_36039236866" + }, + { + "tilingKey": 36037532930, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603_36037532930" + }, + { + "tilingKey": 70935846146, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603_70935846146" + }, + { + "tilingKey": 70934142210, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603_70934142210" + }, + { + "tilingKey": 71472717058, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603_71472717058" + }, + { + "tilingKey": 71471013122, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603_71471013122" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/29,2/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/29,2/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/29,2/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/29,2/0,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "caeeb937450177ae3decff4a8369e1ee663b72480ec5c0e15d09ff7cfd5d4c4c,4eb99686380906190c2e041fd26dc5039d0ecaec520483bb3de5d55afcdc452c", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603.o new file mode 100644 index 00000000..c1430dc6 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc.json new file mode 100644 index 00000000..ffe1815b --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc.json @@ -0,0 +1,463 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "c8db77b4957e1d62c8c64bf2e2384d3d9e7d405a70e2e2d6ce316ff8a8c22c4a", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 34361508354, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34361508354" + }, + { + "tilingKey": 34629943810, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34629943810" + }, + { + "tilingKey": 34428617218, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34428617218" + }, + { + "tilingKey": 34697052674, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34697052674" + }, + { + "tilingKey": 34495726082, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34495726082" + }, + { + "tilingKey": 34764161538, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34764161538" + }, + { + "tilingKey": 34395062786, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34395062786" + }, + { + "tilingKey": 34663498242, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34663498242" + }, + { + "tilingKey": 34462171650, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34462171650" + }, + { + "tilingKey": 34730607106, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34730607106" + }, + { + "tilingKey": 34529280514, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34529280514" + }, + { + "tilingKey": 34797715970, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34797715970" + }, + { + "tilingKey": 34359804418, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34359804418" + }, + { + "tilingKey": 34628239874, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34628239874" + }, + { + "tilingKey": 34426913282, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34426913282" + }, + { + "tilingKey": 34695348738, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34695348738" + }, + { + "tilingKey": 34494022146, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34494022146" + }, + { + "tilingKey": 34762457602, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34762457602" + }, + { + "tilingKey": 34393358850, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34393358850" + }, + { + "tilingKey": 34661794306, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34661794306" + }, + { + "tilingKey": 34460467714, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34460467714" + }, + { + "tilingKey": 34728903170, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34728903170" + }, + { + "tilingKey": 34527576578, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34527576578" + }, + { + "tilingKey": 34796012034, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_34796012034" + }, + { + "tilingKey": 68721246722, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_68721246722" + }, + { + "tilingKey": 68788355586, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_68788355586" + }, + { + "tilingKey": 68754801154, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_68754801154" + }, + { + "tilingKey": 68821910018, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_68821910018" + }, + { + "tilingKey": 68719542786, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_68719542786" + }, + { + "tilingKey": 68786651650, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_68786651650" + }, + { + "tilingKey": 68753097218, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_68753097218" + }, + { + "tilingKey": 68820206082, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_68820206082" + }, + { + "tilingKey": 171798757890, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc_171798757890" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/27,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/27,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/27,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/27,2/27,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "7f9262eeb300688132b2bf28b9f90c76897495e552128d47a991dd28b7238ec8,21b90310f63da323c7e21bf098d7c948a9acf2072957a557f39bfb225f7b0f62", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc.o new file mode 100644 index 00000000..da33c0c1 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd.json new file mode 100644 index 00000000..bd49bc01 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd.json @@ -0,0 +1,463 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "db7e3579a20ec69e0435bd35e7425bae355325d67e7873f2032a1b153e02c782", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 34361508354, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34361508354" + }, + { + "tilingKey": 34629943810, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34629943810" + }, + { + "tilingKey": 34428617218, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34428617218" + }, + { + "tilingKey": 34697052674, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34697052674" + }, + { + "tilingKey": 34495726082, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34495726082" + }, + { + "tilingKey": 34764161538, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34764161538" + }, + { + "tilingKey": 34395062786, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34395062786" + }, + { + "tilingKey": 34663498242, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34663498242" + }, + { + "tilingKey": 34462171650, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34462171650" + }, + { + "tilingKey": 34730607106, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34730607106" + }, + { + "tilingKey": 34529280514, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34529280514" + }, + { + "tilingKey": 34797715970, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34797715970" + }, + { + "tilingKey": 34359804418, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34359804418" + }, + { + "tilingKey": 34628239874, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34628239874" + }, + { + "tilingKey": 34426913282, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34426913282" + }, + { + "tilingKey": 34695348738, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34695348738" + }, + { + "tilingKey": 34494022146, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34494022146" + }, + { + "tilingKey": 34762457602, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34762457602" + }, + { + "tilingKey": 34393358850, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34393358850" + }, + { + "tilingKey": 34661794306, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34661794306" + }, + { + "tilingKey": 34460467714, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34460467714" + }, + { + "tilingKey": 34728903170, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34728903170" + }, + { + "tilingKey": 34527576578, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34527576578" + }, + { + "tilingKey": 34796012034, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_34796012034" + }, + { + "tilingKey": 68721246722, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_68721246722" + }, + { + "tilingKey": 68788355586, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_68788355586" + }, + { + "tilingKey": 68754801154, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_68754801154" + }, + { + "tilingKey": 68821910018, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_68821910018" + }, + { + "tilingKey": 68719542786, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_68719542786" + }, + { + "tilingKey": 68786651650, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_68786651650" + }, + { + "tilingKey": 68753097218, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_68753097218" + }, + { + "tilingKey": 68820206082, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_68820206082" + }, + { + "tilingKey": 171798757890, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd_171798757890" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/3,2/0,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "cbed479e9994a98b31a843855bd9156d8a9b64dfdf3cff06590c5d0997547724,686addb2e3cc88c15cb6b9d55a0fe21419e140a5d7d759babd56baf96b8a09ae", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd.o new file mode 100644 index 00000000..14f0b985 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639.json new file mode 100644 index 00000000..9578be84 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639.json @@ -0,0 +1,463 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "e60d19cf640e0eab141d82833999e57054efdf99a8133ba45c0562d625e160a7", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 34361508354, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34361508354" + }, + { + "tilingKey": 34629943810, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34629943810" + }, + { + "tilingKey": 34428617218, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34428617218" + }, + { + "tilingKey": 34697052674, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34697052674" + }, + { + "tilingKey": 34495726082, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34495726082" + }, + { + "tilingKey": 34764161538, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34764161538" + }, + { + "tilingKey": 34395062786, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34395062786" + }, + { + "tilingKey": 34663498242, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34663498242" + }, + { + "tilingKey": 34462171650, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34462171650" + }, + { + "tilingKey": 34730607106, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34730607106" + }, + { + "tilingKey": 34529280514, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34529280514" + }, + { + "tilingKey": 34797715970, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34797715970" + }, + { + "tilingKey": 34359804418, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34359804418" + }, + { + "tilingKey": 34628239874, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34628239874" + }, + { + "tilingKey": 34426913282, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34426913282" + }, + { + "tilingKey": 34695348738, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34695348738" + }, + { + "tilingKey": 34494022146, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34494022146" + }, + { + "tilingKey": 34762457602, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34762457602" + }, + { + "tilingKey": 34393358850, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34393358850" + }, + { + "tilingKey": 34661794306, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34661794306" + }, + { + "tilingKey": 34460467714, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34460467714" + }, + { + "tilingKey": 34728903170, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34728903170" + }, + { + "tilingKey": 34527576578, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34527576578" + }, + { + "tilingKey": 34796012034, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_34796012034" + }, + { + "tilingKey": 68721246722, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_68721246722" + }, + { + "tilingKey": 68788355586, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_68788355586" + }, + { + "tilingKey": 68754801154, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_68754801154" + }, + { + "tilingKey": 68821910018, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_68821910018" + }, + { + "tilingKey": 68719542786, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_68719542786" + }, + { + "tilingKey": 68786651650, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_68786651650" + }, + { + "tilingKey": 68753097218, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_68753097218" + }, + { + "tilingKey": 68820206082, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_68820206082" + }, + { + "tilingKey": 171798757890, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639_171798757890" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,2/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,2/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,2/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,2/3,2/27,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "6a08dd823f1f9e0f3f19546e7dc653a64fc4332b621c98537e385e87f271f1eb,4c7637d71a7c4a26b96ff967781880b9dbac1eae2fe876a2e4543935d46123d8", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639.o new file mode 100644 index 00000000..db95edbe Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be.json new file mode 100644 index 00000000..f31a5e11 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be.json @@ -0,0 +1,260 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "2eef7878ffa1409cd27b5e3c4663b1b70f03e9cd450f52a12a13ca6015fb72a9", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 68721253661, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be_68721253661" + }, + { + "tilingKey": 68788362525, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be_68788362525" + }, + { + "tilingKey": 68719549725, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be_68719549725" + }, + { + "tilingKey": 68786658589, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be_68786658589" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/29,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/29,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/29,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/29,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "0cf3662aaeedb453dbfc3340f161bc738cae2620c79f0ea70b461f664866bdbb,32abe20720d62a54c8667afc7c6aba0ce2920d2c52d09302dfaef4f326c92a1f", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be.o new file mode 100644 index 00000000..2cf12b8b Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d.json new file mode 100644 index 00000000..a134ed98 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d.json @@ -0,0 +1,463 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "8060deb78a337e571bb3c07e6cbd608c4ab0d041422461a0824554998f397c6e", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 34361508354, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34361508354" + }, + { + "tilingKey": 34629943810, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34629943810" + }, + { + "tilingKey": 34428617218, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34428617218" + }, + { + "tilingKey": 34697052674, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34697052674" + }, + { + "tilingKey": 34495726082, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34495726082" + }, + { + "tilingKey": 34764161538, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34764161538" + }, + { + "tilingKey": 34395062786, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34395062786" + }, + { + "tilingKey": 34663498242, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34663498242" + }, + { + "tilingKey": 34462171650, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34462171650" + }, + { + "tilingKey": 34730607106, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34730607106" + }, + { + "tilingKey": 34529280514, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34529280514" + }, + { + "tilingKey": 34797715970, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34797715970" + }, + { + "tilingKey": 34359804418, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34359804418" + }, + { + "tilingKey": 34628239874, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34628239874" + }, + { + "tilingKey": 34426913282, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34426913282" + }, + { + "tilingKey": 34695348738, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34695348738" + }, + { + "tilingKey": 34494022146, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34494022146" + }, + { + "tilingKey": 34762457602, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34762457602" + }, + { + "tilingKey": 34393358850, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34393358850" + }, + { + "tilingKey": 34661794306, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34661794306" + }, + { + "tilingKey": 34460467714, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34460467714" + }, + { + "tilingKey": 34728903170, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34728903170" + }, + { + "tilingKey": 34527576578, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34527576578" + }, + { + "tilingKey": 34796012034, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_34796012034" + }, + { + "tilingKey": 68721246722, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_68721246722" + }, + { + "tilingKey": 68788355586, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_68788355586" + }, + { + "tilingKey": 68754801154, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_68754801154" + }, + { + "tilingKey": 68821910018, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_68821910018" + }, + { + "tilingKey": 68719542786, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_68719542786" + }, + { + "tilingKey": 68786651650, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_68786651650" + }, + { + "tilingKey": 68753097218, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_68753097218" + }, + { + "tilingKey": 68820206082, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_68820206082" + }, + { + "tilingKey": 171798757890, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d_171798757890" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/27,2/0,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/27,2/0,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/27,2/0,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/27,2/0,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "869a68bdea2dd8420c14782947400dfd7df5cb874ff146d56a88f6fdb4f45571,2472b330dcd83b6ea8be0c9f6481cc59285d4d8f3b8babde3361e63b812a3442", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d.o new file mode 100644 index 00000000..daba7477 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0.json new file mode 100644 index 00000000..29e3e710 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0.json @@ -0,0 +1,316 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "4393aa626d393cc32e490fce09866145983a523acbf7eb73732854b371fee462", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 68721253659, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0_68721253659" + }, + { + "tilingKey": 68788362523, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0_68788362523" + }, + { + "tilingKey": 68719549697, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0_68719549697" + }, + { + "tilingKey": 68786658561, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0_68786658561" + }, + { + "tilingKey": 34361515291, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0_34361515291" + }, + { + "tilingKey": 34428624155, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0_34428624155" + }, + { + "tilingKey": 34395069723, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0_34395069723" + }, + { + "tilingKey": 34462178587, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0_34462178587" + }, + { + "tilingKey": 34359811329, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0_34359811329" + }, + { + "tilingKey": 34426920193, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0_34426920193" + }, + { + "tilingKey": 34393365761, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0_34393365761" + }, + { + "tilingKey": 34460474625, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0_34460474625" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/27,2/29,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/27,2/29,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/27,2/29,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/27,2/29,2/0,2/10,2/0,2/27,2/27,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "1e5148e57556bbb3f5b3b56202bf96fa1385f30013c54515bca1c41bc343cdd8,794827a1fc00d8c156af51c6e630777bd09e99ac23b12ec77aebaf37792007a1", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0.o new file mode 100644 index 00000000..3f6fbc09 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec.json new file mode 100644 index 00000000..401ecf13 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec.json @@ -0,0 +1,421 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "c23cd7a49828521207c26891b031719ec044b55938fc391553df25bba0736de7", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 1776411, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_1776411" + }, + { + "tilingKey": 68885275, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_68885275" + }, + { + "tilingKey": 135994139, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_135994139" + }, + { + "tilingKey": 65793, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_65793" + }, + { + "tilingKey": 67174657, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_67174657" + }, + { + "tilingKey": 134283521, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_134283521" + }, + { + "tilingKey": 0, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_0" + }, + { + "tilingKey": 67108864, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_67108864" + }, + { + "tilingKey": 134217728, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_134217728" + }, + { + "tilingKey": 35330843, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_35330843" + }, + { + "tilingKey": 102439707, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_102439707" + }, + { + "tilingKey": 169548571, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_169548571" + }, + { + "tilingKey": 33620225, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_33620225" + }, + { + "tilingKey": 100729089, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_100729089" + }, + { + "tilingKey": 167837953, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_167837953" + }, + { + "tilingKey": 33554432, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_33554432" + }, + { + "tilingKey": 100663296, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_100663296" + }, + { + "tilingKey": 167772160, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:0", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_167772160" + }, + { + "tilingKey": 34378291995, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_34378291995" + }, + { + "tilingKey": 34445400859, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_34445400859" + }, + { + "tilingKey": 34512509723, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_34512509723" + }, + { + "tilingKey": 34376581377, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_34376581377" + }, + { + "tilingKey": 34443690241, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_34443690241" + }, + { + "tilingKey": 34510799105, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_34510799105" + }, + { + "tilingKey": 34376515584, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_34376515584" + }, + { + "tilingKey": 34443624448, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_34443624448" + }, + { + "tilingKey": 34510733312, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec_34510733312" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/0,2/0,2/0,2/10,2/0,2/1,2/1,2/0,2", + "DlinferGroupedMatmulDirect/d=0,p=0/0,2/0,2/0,2/10,2/0,2/1,2/1,2/0,2", + "DlinferGroupedMatmulDirect/d=1,p=1/0,2/0,2/0,2/10,2/0,2/1,2/1,2/0,2", + "DlinferGroupedMatmulDirect/d=1,p=0/0,2/0,2/0,2/10,2/0,2/1,2/1,2/0,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "1dc62a954f0a0ed5bedd40c3ed7da65bc5f39641047fc31f4c729fd90ccac962,6b7232196fc8ea58b9aa45e108c858b3a29fa25014501455ae4ea9c6b188399b", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec.o new file mode 100644 index 00000000..60c41e00 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a.json new file mode 100644 index 00000000..83279ecc --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a.json @@ -0,0 +1,456 @@ +{ + "binFileName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a", + "binFileSuffix": ".o", + "blockDim": -1, + "coreType": "MIX", + "intercoreSync": 1, + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a", + "magic": "RT_DEV_BINARY_MAGIC_ELF", + "memoryStamping": [], + "opParaSize": 2016, + "parameters": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "sha256": "647285852d21086ed280777ecc59308ae8518345f12db295abee2c200b5b390a", + "workspace": { + "num": 1, + "size": [ + -1 + ], + "type": [ + 0 + ] + }, + "kernelList": [ + { + "tilingKey": 68721253659, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_68721253659" + }, + { + "tilingKey": 68788362523, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_68788362523" + }, + { + "tilingKey": 68719549697, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_68719549697" + }, + { + "tilingKey": 68786658561, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_68786658561" + }, + { + "tilingKey": 34361515291, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_34361515291" + }, + { + "tilingKey": 34428624155, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_34428624155" + }, + { + "tilingKey": 34395069723, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_34395069723" + }, + { + "tilingKey": 34462178587, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_34462178587" + }, + { + "tilingKey": 34359811329, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_34359811329" + }, + { + "tilingKey": 34426920193, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_34426920193" + }, + { + "tilingKey": 34393365761, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_34393365761" + }, + { + "tilingKey": 34460474625, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_34460474625" + }, + { + "tilingKey": 42951442971, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_42951442971" + }, + { + "tilingKey": 51541377563, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_51541377563" + }, + { + "tilingKey": 43018551835, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_43018551835" + }, + { + "tilingKey": 51608486427, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_51608486427" + }, + { + "tilingKey": 42984997403, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_42984997403" + }, + { + "tilingKey": 51574931995, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_51574931995" + }, + { + "tilingKey": 43052106267, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_43052106267" + }, + { + "tilingKey": 51642040859, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_51642040859" + }, + { + "tilingKey": 42949739009, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_42949739009" + }, + { + "tilingKey": 51539673601, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_51539673601" + }, + { + "tilingKey": 43016847873, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_43016847873" + }, + { + "tilingKey": 51606782465, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_51606782465" + }, + { + "tilingKey": 42983293441, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_42983293441" + }, + { + "tilingKey": 51573228033, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_51573228033" + }, + { + "tilingKey": 43050402305, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_43050402305" + }, + { + "tilingKey": 51640336897, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:1", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_51640336897" + }, + { + "tilingKey": 85901115931, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_85901115931" + }, + { + "tilingKey": 85968224795, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_85968224795" + }, + { + "tilingKey": 85899411969, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_85899411969" + }, + { + "tilingKey": 85966520833, + "kernelType": "MIX_AIC", + "crossCoreSync": 1, + "taskRation": "1:2", + "kernelName": "DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a_85966520833" + } + ], + "taskRation": "tilingKey", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "supportSuperKernel": 1, + "compileInfo": {}, + "dynamicParamMode": "folded_with_desc", + "supportInfo": { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/27,2/2,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/27,2/2,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/27,2/2,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/27,2/2,2/0,2/10,2/0,2/27,2/27,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "staticKey": "ac0299366b76bed29b6dbac225c2c251c2a0ec34808e266ad4a6b2582a05290b,6c8ec7c0db2b7c1681223d2abd391f3cb1930e296cd1f1572a40d28b40d142e6", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "deterministic": "ignore" + }, + "filePath": "ascend910_93/bin/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a.json" +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a.o b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a.o new file mode 100644 index 00000000..fda4d924 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a.o differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/config/ascend910_93/binary_info_config.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/config/ascend910_93/binary_info_config.json new file mode 100644 index 00000000..b8b3bf2a --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/config/ascend910_93/binary_info_config.json @@ -0,0 +1,430 @@ +{ + "DlinferGroupedMatmulDirect": { + "dynamicRankSupport": true, + "simplifiedKeyMode": 0, + "optionalInputMode": "gen_placeholder", + "optionalOutputMode": "gen_placeholder", + "params": { + "inputs": [ + [ + { + "name": "x", + "index": 0, + "paramType": "dynamic", + "formatMode": "static_nd_agnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "paramType": "dynamic", + "formatMode": "static_nd_agnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "paramType": "dynamic", + "formatMode": "static_nd_agnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "paramType": "dynamic", + "formatMode": "static_nd_agnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "paramType": "dynamic", + "formatMode": "static_nd_agnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "paramType": "dynamic", + "formatMode": "static_nd_agnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "paramType": "dynamic", + "formatMode": "static_nd_agnostic" + } + ], + { + "name": "group_list", + "index": 7, + "paramType": "optional", + "formatMode": "static_nd_agnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "paramType": "optional", + "formatMode": "static_nd_agnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "paramType": "dynamic", + "formatMode": "static_nd_agnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item" + }, + { + "name": "dtype" + }, + { + "name": "transpose_weight" + }, + { + "name": "transpose_x" + }, + { + "name": "group_type" + }, + { + "name": "group_list_type" + }, + { + "name": "act_type" + }, + { + "name": "tuning_config" + } + ] + }, + "binaryList": [ + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/3,2/10,2/0,2/1,2/1,2/3,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/29,2/29,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/29,2/29,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/29,2/29,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/29,2/29,29/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/27,2/27,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/27,2/27,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/27,2/27,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/27,2/27,29/0,2/10,2/0,2/1,2/1,2/27,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/1,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/1,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/1,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/1,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/29,2/29,29/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/29,2/29,29/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/29,2/29,29/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/29,2/29,29/1,2/10,2/0,2/1,2/1,2/27,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/29,29/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/29,29/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/29,29/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/29,29/0,2/10,2/0,2/1,2/1,2/1,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,2/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,2/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,2/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,2/3,2/10,2/0,2/1,2/1,2/3,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,2/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,2/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,2/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,2/3,2/0,2/0,2/1,2/1,2/1,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,2/3,2/10,2/0,2/1,2/1,2/2,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,2/3,2/10,2/0,2/1,2/1,2/2,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,2/3,2/10,2/0,2/1,2/1,2/2,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,2/3,2/10,2/0,2/1,2/1,2/2,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/1,2/2,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/1,2/2,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/1,2/2,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/1,2/2,2/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/1,2/1,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/1,2/1,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/1,2/1,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/1,2/1,29/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/29,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/29,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/29,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/29,2/0,2/10,2/0,2/1,2/1,2/27,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/3,2/27,2/0,2/1,2/1,2/27,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/27,2/27,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/27,2/27,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/27,2/27,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/27,2/27,2/0,2/10,2/0,2/1,2/1,2/27,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/29,2/29,2/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/29,2/29,2/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/29,2/29,2/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/29,2/29,2/1,2/10,2/0,2/1,2/1,2/27,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/29,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/29,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/29,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/29,29/0,2/10,2/0,2/1,2/1,2/27,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/1,2/1,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/1,2/1,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/1,2/1,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/1,2/1,2/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/29,2/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/29,2/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/29,2/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/29,2/0,2/10,2/0,2/1,2/1,2/1,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/27,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/27,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/27,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/27,2/27,2/0,2/1,2/1,2/27,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/3,2/0,2/0,2/1,2/1,2/1,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,2/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,2/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,2/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,2/3,2/27,2/0,2/1,2/1,2/27,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/29,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/29,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/29,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/29,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/27,2/0,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/27,2/0,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/27,2/0,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/27,2/0,2/0,2/1,2/1,2/27,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/27,2/29,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/27,2/29,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/27,2/29,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/27,2/29,2/0,2/10,2/0,2/27,2/27,2/27,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/0,2/0,2/0,2/10,2/0,2/1,2/1,2/0,2", + "DlinferGroupedMatmulDirect/d=0,p=0/0,2/0,2/0,2/10,2/0,2/1,2/1,2/0,2", + "DlinferGroupedMatmulDirect/d=1,p=1/0,2/0,2/0,2/10,2/0,2/1,2/1,2/0,2", + "DlinferGroupedMatmulDirect/d=1,p=0/0,2/0,2/0,2/10,2/0,2/1,2/1,2/0,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec.json" + }, + { + "coreType": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/27,2/2,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/27,2/2,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/27,2/2,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/27,2/2,2/0,2/10,2/0,2/27,2/27,2/27,2" + ], + "multiKernelType": 1, + "binPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a.o", + "jsonPath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a.json" + } + ] + } +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/config/ascend910_93/dlinfer_grouped_matmul_direct.json b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/config/ascend910_93/dlinfer_grouped_matmul_direct.json new file mode 100644 index 00000000..b6dd405c --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/kernel/config/ascend910_93/dlinfer_grouped_matmul_direct.json @@ -0,0 +1,4996 @@ +{ + "binList": [ + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/3,2/10,2/0,2/1,2/1,2/3,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "8b41b4586c56d0b2acef879821e22a2d5d883c4ad53c549732dcdab2f8ca509c,fa6b402a3f5197c0f0cc15f13817b0399903a58934cf9b4eedfdd70862a9d147", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_0af973f410e50e3e3b1ff41aea54265c.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/29,2/29,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/29,2/29,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/29,2/29,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/29,2/29,29/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "13629ba98b813b7dbab170a22eab0bf06e7285a915cbc129d237325e15b6ba66,d9bf188a31acbb54fdb5028b92225dcac4701b0917b44041c2995023017aa1ad", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_14ea801e0230b6c76bab67d2cd45dd31.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/27,2/27,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/27,2/27,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/27,2/27,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/27,2/27,29/0,2/10,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "d4e39095f87cf6a91932800684b59e22dbac70172f185f4b43e4ee2a4d4a5da5,0ddc3ebcab5f87244f7fb305c17262fdc6568a5358d99481c865f521f6e6e220", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "bfloat16", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_173388342555b3fc6f112c998da052a2.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/1,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/1,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/1,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/1,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "b341252294cb8f5035344367a36a364c6b361e1b9d4fccc79dea908593dcba83,7b05f5a3db1225db3287298ef758c10399ec9d331ad4d64d2f77b9cf1f812a0b", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1a98e21bb0869f65d5a8c91d61b0033b.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/29,2/29,29/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/29,2/29,29/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/29,2/29,29/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/29,2/29,29/1,2/10,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "18a1653b6a965a7b7b23b9969942498b985ccb4631b9090d517ea5878f304f5a,2740b74e9b27b6447f39836565aba3e76317bc29a3d4c4d7e0624990da64122f", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_1b6f67952c45160f3a6117bfd672257c.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/29,29/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/29,29/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/29,29/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/29,29/0,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "a4847c0b99929f7ae30bdcd5dcb5becaeba41561fc1b097028f9e21258768db7,f85725b8721a243ce333e83f9bd0647c8f072d1162577de2c9fd34dd6859c64d", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_25fdaec6c39d32326f15567d56a2e28a.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,2/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,2/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,2/3,2/10,2/0,2/1,2/1,2/3,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,2/3,2/10,2/0,2/1,2/1,2/3,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "8059ba9619837992c932ff680fb253301eaa4c20ea03fc8c4972d8cf67c60687,22bbb045a142913adde66ef6b543c7e71945d61243cdd268fc2c10f14bfef71a", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_2b41b319d91a3be65856f8813f18378f.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,2/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,2/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,2/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,2/3,2/0,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "45aad95117637f87eb605ae9ed2c2879a00fea3f4b749f77b4415ee545c91046,544a277ccb408894775bb088b1a9f718ba38223aafa0823f78ffa51a27e0653b", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_30f8bfbb76700a482746b159d3ea37fb.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,2/3,2/10,2/0,2/1,2/1,2/2,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,2/3,2/10,2/0,2/1,2/1,2/2,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,2/3,2/10,2/0,2/1,2/1,2/2,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,2/3,2/10,2/0,2/1,2/1,2/2,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "739a802e05a38a09071755a4cda95d56820a00674eb74107b484bb956c755be5,b7a249aa6a17cca837a75b0df16d3d1c7cd1fa1664365bb412254c2bc106645f", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_32b6a71f310564e9332c0f75514b7049.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/1,2/2,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/1,2/2,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/1,2/2,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/1,2/2,2/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "a2a29e208da73038001e15c9f5afba2f89b3394b48b702e5418de2635ed8084d,61a5fd1fd70cdb16b149aca1874ecf9d74f9c32a84b8ec3322edd432bf6c0e07", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_38f063a60c111f0a125f7e242d10cccf.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/1,2/1,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/1,2/1,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/1,2/1,29/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/1,2/1,29/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "4fcb19cc155249cee35987efc2a997f60ebb50b7ad0c22120b1237bd975d58eb,5cf37210872c01ad8c8f05d6a8c5d1f42cfbab310986687eb859b1bd802d1e7c", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "float16", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_54e7c7c453eafc8bfd2f8e35ad14e952.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/29,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/29,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/29,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/29,2/0,2/10,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "b78628594c90cbd20276f56387a3fe7d70992796683188dac9585f2f2c3843d0,eb6ba8895861666637b0bc4575c4dba66e669e558e2ff95fa9e721644e74a38d", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_651dac79c82df857a3d4d03bc03a7cf0.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/3,2/27,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "7ee92b880724aef3a3277ca7649a512d291908a635012aa90e015adcaf741903,ac024fac9c65de08068669ae7ea632fcbc14b835feee61f458bde27efc192482", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_693b2681fd562fa85e51f6297bfb5d5a.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/27,2/27,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/27,2/27,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/27,2/27,2/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/27,2/27,2/0,2/10,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "e3fe5fb04d6809c2a00838a1a4bfea5de45c4034c057e608ab6b277da815a07a,fc282608b185db5ba03378f8a51f56cf2e3c06d947dccce5551d62f055cca6dd", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_6a4b53c9c870c698c07eba4c4b35f3c8.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/29,2/29,2/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/29,2/29,2/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/29,2/29,2/1,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/29,2/29,2/1,2/10,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "1ed4764a2067b34de7ad9316df128d7c57f4b86c00310be6ad812b79ca1e56d4,369fb10680882b00990ee64c6a9168a7c35a891b48fa35c532a84505a8cd59a6", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_798a2ceb092836c6d05f0fe9fec9bf5a.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/29,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/29,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/29,29/0,2/10,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/29,29/0,2/10,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "48b2e2a4c56aac6e2ffa2314d3a7e4ff90095eb3952dfdda0e3ed89ac831d60a,1448128e62c4dfbd70471cdb63281ff760f4ded0ff3fd7a5671c6c75308ddc7b", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_86283a1ad10e4c6b780edccce2045ed2.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/1,2/1,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/1,2/1,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/1,2/1,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/1,2/1,2/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "7642f687f619d1419cccee1f466a3624fb1be887b15029a41aab74acab2e247b,3a506a2e7d3168ad349cbf48232cdaa6d81af8783e4a3b179adc1c11a8249b62", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_8c63cc461dff16c68ca75c3349addb45.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/29,2/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/29,2/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/29,2/0,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/29,2/0,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "caeeb937450177ae3decff4a8369e1ee663b72480ec5c0e15d09ff7cfd5d4c4c,4eb99686380906190c2e041fd26dc5039d0ecaec520483bb3de5d55afcdc452c", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_9c97f570647d771ce20bbd181d0d3603.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/27,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/27,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/27,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/27,2/27,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "7f9262eeb300688132b2bf28b9f90c76897495e552128d47a991dd28b7238ec8,21b90310f63da323c7e21bf098d7c948a9acf2072957a557f39bfb225f7b0f62", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_a002667212e4f3701fcc94ba4ec937fc.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/3,2/0,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/3,2/0,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "cbed479e9994a98b31a843855bd9156d8a9b64dfdf3cff06590c5d0997547724,686addb2e3cc88c15cb6b9d55a0fe21419e140a5d7d759babd56baf96b8a09ae", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_d5c9aa8fc65613bf4b96a3255c73f0bd.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,2/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,2/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,2/3,2/27,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,2/3,2/27,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "6a08dd823f1f9e0f3f19546e7dc653a64fc4332b621c98537e385e87f271f1eb,4c7637d71a7c4a26b96ff967781880b9dbac1eae2fe876a2e4543935d46123d8", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "int32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_dc06464e9cd662577a331629f094b639.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/29,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=0,p=0/29,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=1/29,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2", + "DlinferGroupedMatmulDirect/d=1,p=0/29,2/29,2/1,2/10,2/0,2/1,2/1,2/1,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "0cf3662aaeedb453dbfc3340f161bc738cae2620c79f0ea70b461f664866bdbb,32abe20720d62a54c8667afc7c6aba0ce2920d2c52d09302dfaef4f326c92a1f", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_eafa4417edec4bfc453e0f2ee6c780be.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/2,2/2,29/27,2/0,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/2,2/2,29/27,2/0,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/2,2/2,29/27,2/0,2/0,2/1,2/1,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/2,2/2,29/27,2/0,2/0,2/1,2/1,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "869a68bdea2dd8420c14782947400dfd7df5cb874ff146d56a88f6fdb4f45571,2472b330dcd83b6ea8be0c9f6481cc59285d4d8f3b8babde3361e63b812a3442", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "FRACTAL_NZ", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ee878b42b4770d3e18f321c0a424d90d.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/27,2/29,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/27,2/29,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/27,2/29,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/27,2/29,2/0,2/10,2/0,2/27,2/27,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "1e5148e57556bbb3f5b3b56202bf96fa1385f30013c54515bca1c41bc343cdd8,794827a1fc00d8c156af51c6e630777bd09e99ac23b12ec77aebaf37792007a1", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int4", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_f188226a1116f53431714694c30721e0.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/0,2/0,2/0,2/10,2/0,2/1,2/1,2/0,2", + "DlinferGroupedMatmulDirect/d=0,p=0/0,2/0,2/0,2/10,2/0,2/1,2/1,2/0,2", + "DlinferGroupedMatmulDirect/d=1,p=1/0,2/0,2/0,2/10,2/0,2/1,2/1,2/0,2", + "DlinferGroupedMatmulDirect/d=1,p=0/0,2/0,2/0,2/10,2/0,2/1,2/1,2/0,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "1dc62a954f0a0ed5bedd40c3ed7da65bc5f39641047fc31f4c729fd90ccac962,6b7232196fc8ea58b9aa45e108c858b3a29fa25014501455ae4ea9c6b188399b", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "float16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_fc8a39813d19017ede290022206c2fec.json" + } + }, + { + "implMode": "high_performance", + "int64Mode": false, + "simplifiedKeyMode": 0, + "simplifiedKey": [ + "DlinferGroupedMatmulDirect/d=0,p=1/27,2/2,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=0,p=0/27,2/2,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=1/27,2/2,2/0,2/10,2/0,2/27,2/27,2/27,2", + "DlinferGroupedMatmulDirect/d=1,p=0/27,2/2,2/0,2/10,2/0,2/27,2/27,2/27,2" + ], + "dynamicParamMode": "folded_with_desc", + "staticKey": "ac0299366b76bed29b6dbac225c2c251c2a0ec34808e266ad4a6b2582a05290b,6c8ec7c0db2b7c1681223d2abd391f3cb1930e296cd1f1572a40d28b40d142e6", + "inputs": [ + [ + { + "name": "x", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "weight", + "index": 1, + "dtype": "int8", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "bias", + "index": 2, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "scale", + "index": 3, + "dtype": "uint64", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "offset", + "index": 4, + "dtype": "float32", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_scale", + "index": 5, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + [ + { + "name": "antiquant_offset", + "index": 6, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + { + "name": "group_list", + "index": 7, + "dtype": "int64", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + }, + { + "name": "per_token_scale", + "index": 8, + "dtype": "float32", + "format": "ND", + "paramType": "optional", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ], + "outputs": [ + [ + { + "name": "y", + "index": 0, + "dtype": "bfloat16", + "format": "ND", + "paramType": "dynamic", + "shape": [ + -2 + ], + "format_match_mode": "FormatAgnostic" + } + ] + ], + "attrs": [ + { + "name": "split_item", + "dtype": "int", + "value": 0 + }, + { + "name": "dtype", + "dtype": "int", + "value": 0 + }, + { + "name": "transpose_weight", + "dtype": "bool", + "value": false + }, + { + "name": "transpose_x", + "dtype": "bool", + "value": false + }, + { + "name": "group_type", + "dtype": "int", + "value": 0 + }, + { + "name": "group_list_type", + "dtype": "int", + "value": 0 + }, + { + "name": "act_type", + "dtype": "int", + "value": 0 + }, + { + "name": "tuning_config", + "dtype": "list_int", + "value": [] + } + ], + "opMode": "dynamic", + "optionalInputMode": "gen_placeholder", + "deterministic": "ignore", + "optionalOutputMode": "gen_placeholder", + "binInfo": { + "jsonFilePath": "ascend910_93/dlinfer_grouped_matmul_direct/DlinferGroupedMatmulDirect_ffc5799737aef33466523ffb32ef924a.json" + } + } + ] +} \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/op_master_device/lib/libcust_opmaster.so b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/op_master_device/lib/libcust_opmaster.so new file mode 100644 index 00000000..27ade78b Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/op_master_device/lib/libcust_opmaster.so differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/op_tiling/lib/linux/aarch64/libcust_opmaster_rt2.0.so b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/op_tiling/lib/linux/aarch64/libcust_opmaster_rt2.0.so new file mode 100755 index 00000000..ea2f5866 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/op_tiling/lib/linux/aarch64/libcust_opmaster_rt2.0.so differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/op_tiling/liboptiling.so b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/op_tiling/liboptiling.so new file mode 120000 index 00000000..a35ff4da --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_impl/ai_core/tbe/op_tiling/liboptiling.so @@ -0,0 +1 @@ +lib/linux/aarch64/libcust_opmaster_rt2.0.so \ No newline at end of file diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_proto/lib/linux/aarch64/libcust_opsproto_rt2.0.so b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_proto/lib/linux/aarch64/libcust_opsproto_rt2.0.so new file mode 100755 index 00000000..f7459b53 Binary files /dev/null and b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/op_proto/lib/linux/aarch64/libcust_opsproto_rt2.0.so differ diff --git a/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/version.info b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/version.info new file mode 100644 index 00000000..7f8896a7 --- /dev/null +++ b/dlinfer/vendor/ascend/csrc/grouped_matmul_direct/vendor/version.info @@ -0,0 +1 @@ +custom_opp_compiler_version=8.5.0 diff --git a/dlinfer/vendor/ascend/grouped_matmul_direct.py b/dlinfer/vendor/ascend/grouped_matmul_direct.py new file mode 100644 index 00000000..8d4d3458 --- /dev/null +++ b/dlinfer/vendor/ascend/grouped_matmul_direct.py @@ -0,0 +1,42 @@ +"""Bundled Ascend GroupedMatmul direct2560 extension.""" + +from typing import Optional + +import torch +import torch_npu + +_EXTENSION_IMPORT_ERROR: Optional[BaseException] = None + +try: + from . import _grouped_matmul_direct +except (ImportError, OSError) as exc: + _grouped_matmul_direct = None + _EXTENSION_IMPORT_ERROR = exc + + +def is_available() -> bool: + """Whether the bundled pybind extension and op-api loaded successfully.""" + return _grouped_matmul_direct is not None + + +def unavailable_reason() -> str: + if _EXTENSION_IMPORT_ERROR is None: + return "" + return str(_EXTENSION_IMPORT_ERROR) + + +def grouped_matmul( + x: torch.Tensor, + weight: torch.Tensor, + group_list: torch.Tensor, + group_list_type: int = 1, +) -> torch.Tensor: + if _grouped_matmul_direct is None: + raise RuntimeError( + "DLInfer bundled GroupedMatmul extension is unavailable: " + f"{unavailable_reason()}" + ) + stream_handle = torch_npu.npu.current_stream(x.device).npu_stream + return _grouped_matmul_direct.grouped_matmul( + x, weight, group_list, group_list_type, stream_handle + ) diff --git a/dlinfer/vendor/ascend/moe.py b/dlinfer/vendor/ascend/moe.py index a0a66033..a16ceb4e 100644 --- a/dlinfer/vendor/ascend/moe.py +++ b/dlinfer/vendor/ascend/moe.py @@ -2,6 +2,7 @@ import torch import torch.distributed as dist from dlinfer.utils.type_annotation import MoECommType +from dlinfer.vendor.ascend import grouped_matmul_direct # aclnnGroupedMatmulV5 requires the groupList tensor to have at most 1024 @@ -14,6 +15,14 @@ # True -> catch-all (identical scheme to graph capture) # False -> per-chunk row slicing (weight views, each row computed once) _MOE_PREFILL_USE_CATCHALL = os.environ.get("DLINFER_MOE_PREFILL_CATCHALL", "0") == "1" +_GMM_EXPERIMENT_MODE = os.environ.get("DLINFER_GMM_EXPERIMENT", "chunked") +_GMM_EXPERIMENT_ALLOWED_MODES = {"chunked", "direct2560"} +_GMM_EXPERIMENT_PATH_PRINTED = False +if _GMM_EXPERIMENT_MODE not in _GMM_EXPERIMENT_ALLOWED_MODES: + raise RuntimeError( + "DLINFER_GMM_EXPERIMENT must be set before Python starts to one of " + f"{sorted(_GMM_EXPERIMENT_ALLOWED_MODES)}, got {_GMM_EXPERIMENT_MODE!r}" + ) class ChunkedMoeWeightLayout: @@ -35,6 +44,17 @@ def build_chunked_moe_storage_layout(num_experts: int): if num_experts <= MAX_GROUP_LIST_SIZE: return num_experts, None + # Select the physical expert-weight layout before checkpoint loading and + # graph capture. Direct mode keeps exactly one continuous 2560-row tensor. + use_direct = _GMM_EXPERIMENT_MODE == "direct2560" + if use_direct: + if num_experts != 2560: + raise RuntimeError( + "DLINFER_GMM_EXPERIMENT=direct2560 is restricted to exactly " + f"2560 logical experts, got {num_experts}" + ) + return num_experts, None + chunk_size = MAX_GROUP_LIST_SIZE - 2 num_chunks = (num_experts + chunk_size - 1) // chunk_size storage_num_experts = num_experts + 2 * num_chunks @@ -81,29 +101,43 @@ def _grouped_mlp( down_weights: torch.Tensor, group_list: torch.Tensor, group_list_type: int, + use_bundled_direct: bool = False, ): + grouped_matmul = ( + grouped_matmul_direct.grouped_matmul + if use_bundled_direct + else lambda x, weight, groups, list_type: torch.ops.npu.npu_grouped_matmul( + [x], + [weight], + group_list=groups, + split_item=2, + group_type=0, + group_list_type=list_type, + )[0] + ) + # up sample - up_proj = torch.ops.npu.npu_grouped_matmul( - [hidden_states], - [gate_up_weights.transpose(1, 2)], - group_list=group_list, - split_item=2, - group_type=0, - group_list_type=group_list_type, - )[0] + up_weight = ( + gate_up_weights if use_bundled_direct else gate_up_weights.transpose(1, 2) + ) + up_proj = grouped_matmul( + hidden_states, + up_weight, + group_list, + group_list_type, + ) # activation gate_cache = torch.ops.npu.npu_swiglu(up_proj, -1) # down sample - down_proj = torch.ops.npu.npu_grouped_matmul( - [gate_cache], - [down_weights.transpose(1, 2)], - group_list=group_list, - split_item=2, - group_type=0, - group_list_type=group_list_type, - )[0] + down_weight = down_weights if use_bundled_direct else down_weights.transpose(1, 2) + down_proj = grouped_matmul( + gate_cache, + down_weight, + group_list, + group_list_type, + ) return down_proj @@ -237,6 +271,8 @@ def apply_mlp( group_list_type: int = 1, chunked_moe_layout: ChunkedMoeWeightLayout = None, ): + global _GMM_EXPERIMENT_PATH_PRINTED + num_experts = ( chunked_moe_layout.num_experts if chunked_moe_layout is not None @@ -247,6 +283,76 @@ def apply_mlp( hidden_states, gate_up_weights, down_weights, group_list, group_list_type ) + use_direct = _GMM_EXPERIMENT_MODE == "direct2560" + if use_direct: + if not grouped_matmul_direct.is_available(): + raise RuntimeError( + "DLINFER_GMM_EXPERIMENT=direct2560 requires the bundled " + "GroupedMatmul extension: " + f"{grouped_matmul_direct.unavailable_reason()}" + ) + if num_experts != 2560: + raise RuntimeError(f"direct2560 requires 2560 experts, got {num_experts}") + if chunked_moe_layout is not None: + raise RuntimeError("direct2560 received a packed chunked weight layout") + if gate_up_weights.dim() != 3 or down_weights.dim() != 3: + raise RuntimeError("direct2560 requires rank-3 single-weight tensors") + if gate_up_weights.size(0) != 2560 or down_weights.size(0) != 2560: + raise RuntimeError( + "direct2560 requires continuous 2560-row weights, got " + f"gate_up={gate_up_weights.size(0)}, down={down_weights.size(0)}" + ) + if group_list_type != 1: + raise RuntimeError( + f"direct2560 requires group_list_type=1, got {group_list_type}" + ) + if group_list.dim() != 1 or group_list.shape[0] != 2560: + raise RuntimeError( + "direct2560 requires a one-dimensional 2560-entry group_list, " + f"got {tuple(group_list.shape)}" + ) + supported_dtypes = (torch.bfloat16, torch.float16) + if gate_up_weights.dtype not in supported_dtypes: + raise RuntimeError(f"direct2560 unsupported dtype {gate_up_weights.dtype}") + if ( + down_weights.dtype != gate_up_weights.dtype + or hidden_states.dtype != gate_up_weights.dtype + ): + raise RuntimeError("direct2560 hidden/gate_up/down dtypes must match") + if any( + tensor.device.type != "npu" + for tensor in (hidden_states, gate_up_weights, down_weights, group_list) + ): + raise RuntimeError( + "direct2560 requires hidden, weights and group_list on NPU" + ) + if not _GMM_EXPERIMENT_PATH_PRINTED: + _GMM_EXPERIMENT_PATH_PRINTED = True + print( + "DLINFER GMM experiment: direct2560 " + f"logical_experts={num_experts} weight_rows={gate_up_weights.size(0)} " + "packed=False group_list_type=1 backend=bundled_pybind", + flush=True, + ) + return _grouped_mlp( + hidden_states, + gate_up_weights, + down_weights, + group_list, + group_list_type=1, + use_bundled_direct=True, + ) + + if not _GMM_EXPERIMENT_PATH_PRINTED: + _GMM_EXPERIMENT_PATH_PRINTED = True + print( + "DLINFER GMM experiment: chunked " + f"logical_experts={num_experts} weight_rows={gate_up_weights.size(0)} " + f"packed={getattr(chunked_moe_layout, 'packed', False)} " + f"chunk_size={getattr(chunked_moe_layout, 'chunk_size', None)}", + flush=True, + ) + # More experts than aclnnGroupedMatmulV5 supports: split into chunks of at # most MAX_GROUP_LIST_SIZE groups. Work in per-expert token counts # (group_list_type=1) regardless of the incoming layout. diff --git a/tests/test_grouped_matmul_direct.py b/tests/test_grouped_matmul_direct.py new file mode 100644 index 00000000..6696b7cf --- /dev/null +++ b/tests/test_grouped_matmul_direct.py @@ -0,0 +1,56 @@ +import pytest + +from dlinfer.vendor.ascend import grouped_matmul_direct +import torch + + +torch_npu = pytest.importorskip("torch_npu") + + +@pytest.mark.skipif(not torch_npu.npu.is_available(), reason="requires Ascend NPU") +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +def test_grouped_matmul_direct2560_matches_chunked(dtype): + assert ( + grouped_matmul_direct.is_available() + ), grouped_matmul_direct.unavailable_reason() + + experts = 2560 + hidden_size = 16 + output_size = 16 + tokens = 8 + device = torch.device("npu:0") + + x = torch.randn(tokens, hidden_size, device=device, dtype=dtype) + # Production stores weights as [E, N, K] and passes a transposed view. + stored_weight = torch.randn( + experts, output_size, hidden_size, device=device, dtype=dtype + ) + weight = stored_weight.transpose(1, 2) + group_list = torch.zeros(experts, device=device, dtype=torch.int64) + group_list[:tokens] = 1 + + direct = grouped_matmul_direct.grouped_matmul(x, stored_weight, group_list, 1) + + references = [] + row_start = 0 + expert_start = 0 + while expert_start < experts: + expert_end = min(expert_start + 1024, experts) + chunk_groups = group_list[expert_start:expert_end] + row_end = row_start + int(chunk_groups.sum().cpu()) + if row_end > row_start: + references.append( + torch.ops.npu.npu_grouped_matmul( + [x[row_start:row_end]], + [weight[expert_start:expert_end]], + group_list=chunk_groups, + split_item=2, + group_type=0, + group_list_type=1, + )[0] + ) + row_start = row_end + expert_start = expert_end + + reference = torch.cat(references, dim=0) + torch.testing.assert_close(direct, reference, rtol=0, atol=0)