算子开发时:
op_host侧:rnnMatmul1.SetDim(rnnParams.sysAicCoreNum);
matmul初始化方式:inputMM.Init(inputMMTiling, &pipe);
计算:if ASCEND_IS_AIC { processInputMM();}
processInputMM(){
if (GetBlockIdx() < inputMMTiling.usedCoreNum) {
// printf("inputOffsets.AOffset: %d\n", inputOffsets.AOffset);
// printf("inputOffsets.BOffset: %d\n", inputOffsets.BOffset);
// printf("inputOffsets.COffset: %d\n", inputOffsets.COffset);
inputMM.SetTensorA(xGm[inputOffsets.AOffset]);
// DumpTensor(xGm[inputOffsets.AOffset], __LINE__, 16);
inputMM.SetTensorB(weightGm[inputOffsets.BOffset]);
// DumpTensor(weightGm[inputOffsets.BOffset], __LINE__, 16);
inputMM.SetBias(biasGm[inputOffsets.BOffset]);
// DumpTensor(biasGm[inputOffsets.BOffset], __LINE__, 16);
if (inputTail.nCoreIndx == inputTail.notTailNCoreCount && inputTail.mCoreIndx == inputTail.notTailMCoreCount) {
inputMM.SetTail(inputTail.tailSingleCoreM, inputTail.tailSingleCoreN);
} else if (inputTail.nCoreIndx == inputTail.notTailNCoreCount) {
inputMM.SetTail(inputMMTiling.singleCoreM, inputTail.tailSingleCoreN);
} else if (inputTail.mCoreIndx == inputTail.notTailMCoreCount) {
inputMM.SetTail(inputTail.tailSingleCoreM, inputMMTiling.singleCoreN);
}
inputMM.IterateAll(outputGm.workspace[inputOffsets.COffset], false);
// DumpTensor(outputGm.workspace[inputOffsets.COffset], __LINE__, 16);
}
}
DumpTensor不开启时,执行算子报错信息如下:
EZ9999: Inner Error!
EZ9999: [PID: 37099] 2024-10-17-03:04:47.202.186 The error from device(chipId:1, dieId:0), serial number is 36, there is an fftsplus aivector error exception, core id is 0, error code = 0, dump info: pc start: 0x12c0c00d146c, current: 0x12c0c00d19a8, vec error info: 0xa10114de18, mte error info: 0xfbf65c754f, ifu error info: 0x212c136c11000, ccu error info: 0xe46730102300005c, cube error info: 0, biu error info: 0, aic error mask: 0x6500020bd00028c, para base: 0x12c1000d0080.[FUNC:ProcessStarsCoreErrorInfo][FILE:device_error_proc.cc][LINE:1218]
TraceBack (most recent call last):
The extend info: errcode:(0, 0x200000000000000, 0) errorStr: The MPU address access is invalid. fixp_error0 info: 0x65c754f, fixp_error1 info: 0xfb fsmId:0, tslot:0, thread:0, ctxid:0, blk:9, sublk:1, subErrType:4.[FUNC:ProcessStarsCoreErrorInfo][FILE:device_error_proc.cc][LINE:1230]
Kernel task happen error, retCode=0x26, [aicore exception].[FUNC:PreCheckTaskErr][FILE:davinic_kernel_task.cc][LINE:1221]
AICORE Kernel task happen error, retCode=0x26.[FUNC:GetError][FILE:stream.cc][LINE:1062]
Aicore kernel execute failed, device_id=9, stream_id=2, report_stream_id=2, task_id=0, flip_num=0, fault kernel_name=SelfDynamicRNN_161fc099eb0c35971672981fb05c101e_10000006_mix_aic, fault kernel info ext=none, program id=0, hash=14702801425534950479.[FUNC:GetError][FILE:stream.cc][LINE:1062]
[AIC_INFO] after execute:args print end[FUNC:GetError][FILE:stream.cc][LINE:1062]
rtStreamSynchronize execute failed, reason=[aicore exception][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:53]
synchronize stream failed, runtime result = 507015[FUNC:ReportCallError][FILE:log_inner.cpp][LINE:161]
aclrtSynchronizeStream failed. ERROR: 507015
DumpTensor开启:
inputOffsets.AOffset,inputOffsets.BOffset,inputOffsets.COffset,TensorA,TensorB,Bias数据打印均符合预期,DumpTensor(outputGm.workspace[inputOffsets.COffset], __LINE__, 16)打印值均为零
请问这个问题如何解决??
算子开发时:
op_host侧:rnnMatmul1.SetDim(rnnParams.sysAicCoreNum);
matmul初始化方式:inputMM.Init(inputMMTiling, &pipe);
计算:if ASCEND_IS_AIC { processInputMM();}
processInputMM(){
if (GetBlockIdx() < inputMMTiling.usedCoreNum) {
// printf("inputOffsets.AOffset: %d\n", inputOffsets.AOffset);
// printf("inputOffsets.BOffset: %d\n", inputOffsets.BOffset);
// printf("inputOffsets.COffset: %d\n", inputOffsets.COffset);
inputMM.SetTensorA(xGm[inputOffsets.AOffset]);
// DumpTensor(xGm[inputOffsets.AOffset], __LINE__, 16);
inputMM.SetTensorB(weightGm[inputOffsets.BOffset]);
// DumpTensor(weightGm[inputOffsets.BOffset], __LINE__, 16);
inputMM.SetBias(biasGm[inputOffsets.BOffset]);
// DumpTensor(biasGm[inputOffsets.BOffset], __LINE__, 16);
if (inputTail.nCoreIndx == inputTail.notTailNCoreCount && inputTail.mCoreIndx == inputTail.notTailMCoreCount) {
inputMM.SetTail(inputTail.tailSingleCoreM, inputTail.tailSingleCoreN);
} else if (inputTail.nCoreIndx == inputTail.notTailNCoreCount) {
inputMM.SetTail(inputMMTiling.singleCoreM, inputTail.tailSingleCoreN);
} else if (inputTail.mCoreIndx == inputTail.notTailMCoreCount) {
inputMM.SetTail(inputTail.tailSingleCoreM, inputMMTiling.singleCoreN);
}
inputMM.IterateAll(outputGm.workspace[inputOffsets.COffset], false);
// DumpTensor(outputGm.workspace[inputOffsets.COffset], __LINE__, 16);
}
}
DumpTensor不开启时,执行算子报错信息如下:
EZ9999: Inner Error!
EZ9999: [PID: 37099] 2024-10-17-03:04:47.202.186 The error from device(chipId:1, dieId:0), serial number is 36, there is an fftsplus aivector error exception, core id is 0, error code = 0, dump info: pc start: 0x12c0c00d146c, current: 0x12c0c00d19a8, vec error info: 0xa10114de18, mte error info: 0xfbf65c754f, ifu error info: 0x212c136c11000, ccu error info: 0xe46730102300005c, cube error info: 0, biu error info: 0, aic error mask: 0x6500020bd00028c, para base: 0x12c1000d0080.[FUNC:ProcessStarsCoreErrorInfo][FILE:device_error_proc.cc][LINE:1218]
TraceBack (most recent call last):
The extend info: errcode:(0, 0x200000000000000, 0) errorStr: The MPU address access is invalid. fixp_error0 info: 0x65c754f, fixp_error1 info: 0xfb fsmId:0, tslot:0, thread:0, ctxid:0, blk:9, sublk:1, subErrType:4.[FUNC:ProcessStarsCoreErrorInfo][FILE:device_error_proc.cc][LINE:1230]
Kernel task happen error, retCode=0x26, [aicore exception].[FUNC:PreCheckTaskErr][FILE:davinic_kernel_task.cc][LINE:1221]
AICORE Kernel task happen error, retCode=0x26.[FUNC:GetError][FILE:stream.cc][LINE:1062]
Aicore kernel execute failed, device_id=9, stream_id=2, report_stream_id=2, task_id=0, flip_num=0, fault kernel_name=SelfDynamicRNN_161fc099eb0c35971672981fb05c101e_10000006_mix_aic, fault kernel info ext=none, program id=0, hash=14702801425534950479.[FUNC:GetError][FILE:stream.cc][LINE:1062]
[AIC_INFO] after execute:args print end[FUNC:GetError][FILE:stream.cc][LINE:1062]
rtStreamSynchronize execute failed, reason=[aicore exception][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:53]
synchronize stream failed, runtime result = 507015[FUNC:ReportCallError][FILE:log_inner.cpp][LINE:161]
aclrtSynchronizeStream failed. ERROR: 507015
DumpTensor开启:
inputOffsets.AOffset,inputOffsets.BOffset,inputOffsets.COffset,TensorA,TensorB,Bias数据打印均符合预期,DumpTensor(outputGm.workspace[inputOffsets.COffset], __LINE__, 16)打印值均为零
请问这个问题如何解决??