Atlas300I Pro RuntimeError: call aclnnBatchNorm failed, detail:EZ9999: Inner Error! Parse dynamic kernel config fail.
收藏回复举报
Atlas300I Pro RuntimeError: call aclnnBatchNorm failed, detail:EZ9999: Inner Error! Parse dynamic kernel config fail.
t('forum.solved') 已解决
新人帖
发表于2026-01-05 20:00:25
0 查看

一、宿主机驱动信息

[root@localhost ~]# npu-smi info

+--------------------------------------------------------------------------------------------------------+

| npu-smi 24.1.rc1                                 Version: 24.1.rc1                                     |

+-------------------------------+-----------------+------------------------------------------------------+

| NPU     Name                  | Health          | Power(W)     Temp(C)           Hugepages-Usage(page) |

| Chip    Device                | Bus-Id          | AICore(%)    Memory-Usage(MB)                        |

+===============================+=================+======================================================+

| 2       310P3                 | OK              | NA           57                0     / 0             |

| 0       0                     | 0000:01:00.0    | 0            1837 / 21527                            |

+===============================+=================+======================================================+

| 3       310P3                 | OK              | NA           51                0     / 0             |

| 0       1                     | 0000:03:00.0    | 0            1786 / 21527                            |

+===============================+=================+======================================================+

+-------------------------------+-----------------+------------------------------------------------------+

| NPU     Chip                  | Process id      | Process name             | Process memory(MB)        |

+===============================+=================+======================================================+

| No running processes found in NPU 2                                                                    |

+===============================+=================+======================================================+

| No running processes found in NPU 3                                                                    |

+===============================+=================+======================================================+

官方镜像信息如下

[root@localhost ~]# docker images

REPOSITORY       TAG                             IMAGE ID       CREATED         SIZE

nginx            latest                          d0ff53bf11ef   7 months ago    192MB

ascend-pytorch   24.0.RC3-A2-2.1.0-ubuntu20.04   bbaeeacf95f5   13 months ago   11.3GB

ascend-pytorch   24.0.RC2-A2-2.1.0-ubuntu20.04   8f1cb37dd88c   13 months ago   10.9GB

二、容器启动

docker run -e TZ=Asia/Shanghai -itd -p 9020:9020 --name dzwl_cv0 --privileged --net=host \

--user root \

--device=/dev/davinci0 \

--device=/dev/davinci_manager \

--device=/dev/devmm_svm \

--device=/dev/hisi_hdc \

-v /usr/share/zoneinfo/Asia/Shanghai:/usr/share/zoneinfo/Asia/Shanghai:ro -v /etc/localtime:/etc/localtime:ro \

-v /usr/local/dcmi:/usr/local/dcmi \

-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \

-v /usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/common \

-v /usr/local/Ascend/driver/lib64/driver:/usr/local/Ascend/driver/lib64/driver \

-v /etc/ascend_install.info:/etc/ascend_install.info \

-v /etc/vnpu.cfg:/etc/vnpu.cfg \

-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \

-v /data/media:/data/media \

-v /opt/dzwl_app:/app \

ascend-pytorch:24.0.RC3-A2-2.1.0-ubuntu20.04 python /app/run.py

三、容器内部运行环境

root@localhost:/home/HwHiAiUser# python3 -c "import torch;import torch_npu; a = torch.randn(3, 4).npu(); print(a + a);"

[W compiler_depend.ts:137] Warning: Warning: Device do not support double dtype now, dtype cast repalce with float. (function operator())

.tensor([[-0.7011, -0.2953,  1.3998, -1.5577],

        [ 1.3079,  1.6953, -0.6101, -1.0200],

        [-2.2365, -1.7546, -2.0615,  1.3509]], device='npu:0')

root@localhost:/home/HwHiAiUser# cat /usr/local/Ascend/ascend-toolkit/latest/version.cfg

# version: 1.0

runtime_running_version=[7.5.0.1.129:8.0.RC3]

compiler_running_version=[7.5.0.1.129:8.0.RC3]

hccl_running_version=[7.5.0.1.129:8.0.RC3]

opp_running_version=[7.5.0.1.129:8.0.RC3]

toolkit_running_version=[7.5.0.1.129:8.0.RC3]

aoe_running_version=[7.5.0.1.129:8.0.RC3]

ncs_running_version=[7.5.0.1.129:8.0.RC3]

opp_kernel_running_version=[7.5.0.1.129:8.0.RC3]

runtime_upgrade_version=[7.5.0.1.129:8.0.RC3]

compiler_upgrade_version=[7.5.0.1.129:8.0.RC3]

hccl_upgrade_version=[7.5.0.1.129:8.0.RC3]

opp_upgrade_version=[7.5.0.1.129:8.0.RC3]

toolkit_upgrade_version=[7.5.0.1.129:8.0.RC3]

aoe_upgrade_version=[7.5.0.1.129:8.0.RC3]

ncs_upgrade_version=[7.5.0.1.129:8.0.RC3]

opp_kernel_upgrade_version=[7.5.0.1.129:8.0.RC3]

runtime_installed_version=[7.5.0.1.129:8.0.RC3]

compiler_installed_version=[7.5.0.1.129:8.0.RC3]

hccl_installed_version=[7.5.0.1.129:8.0.RC3]

opp_installed_version=[7.5.0.1.129:8.0.RC3]

toolkit_installed_version=[7.5.0.1.129:8.0.RC3]

aoe_installed_version=[7.5.0.1.129:8.0.RC3]

ncs_installed_version=[7.5.0.1.129:8.0.RC3]

opp_kernel_installed_version=[7.5.0.1.129:8.0.RC3]

root@localhost:/home/HwHiAiUser# 

算子情况

/usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/impl/batch_norm3d.py       /usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/impl/batch_norm_grad.py

/usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/impl/batch_norm3d_grad.py  /usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/impl/batch_norm_grad_ext2.py

root@localhost:/app/yololv9# ls /usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/impl/tensor_move*

/usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/impl/tensor_move.py

root@localhost:/app/yololv9# # 测试 BatchNorm 算子

root@localhost:/app/yololv9# python -c "import torch; import torch_npu; bn = torch.nn.BatchNorm2d(3).npu(); x = torch.randn(1,3,64,64).npu(); y = bn(x); print(y.shape)"

torch.Size([1, 3, 64, 64])

.root@localhost:/app/yololv9# 

root@localhost:/app/yololv9# # 测试 TensorMove 算子

root@localhost:/app/yololv9# python -c "import torch; import torch_npu; x = torch.randn(1,3,64,64).npu(); y = x.moveaxis(1, 3); print(y.shape)"

torch.Size([1, 64, 64, 3])

root@localhost:/app/yololv9# # 检查算子文件权限

root@localhost:/app/yololv9# ls -l /usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/impl/batch_norm.py

-r-xr-xr-x 1 root root 26310 Nov 29  2024 /usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/impl/batch_norm.py

root@localhost:/app/yololv9# ls -l /usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/impl/tensor_move.py

-r-xr-xr-x 1 root root 9323 Nov 29  2024 /usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/impl/tensor_move.py

四、目标代码运行报错

官网源码:https://gitee.com/ascend/ModelZoo-PyTorch/blob/master/PyTorch/built-in/cv/detection/YOLOV9_for_PyTorch/README.md

root@localhost:/home/HwHiAiUser# cd /app/yololv9/

root@localhost:/app/yololv9# python detect.py --source '/app/resource/img/test.png' --img 640 --device 0 --weights '/app/resource/weights/yolov9-m.pt' --name yolov9_c_c_640_detect

detect: weights=['/app/resource/weights/yolov9-m.pt'], source=/app/resource/img/test.png, data=data/coco128.yaml, imgsz=[640, 640], conf_thres=0.25, iou_thres=0.45, max_det=1000, device=0, view_img=False, save_txt=False, save_conf=False, save_crop=False, nosave=False, classes=None, agnostic_nms=False, augment=False, visualize=False, update=False, project=runs/detect, name=yolov9_c_c_640_detect, exist_ok=False, line_thickness=3, hide_labels=False, hide_conf=False, half=False, dnn=False, vid_stride=1

YOLO 🚀 2025-12-30 Python-3.9.2 torch-2.1.0 NPU

/usr/local/python3.9.2/lib/python3.9/site-packages/torch_npu/contrib/transfer_to_npu.py:301: ImportWarning: 

    *************************************************************************************************************

    The torch.Tensor.cuda and torch.nn.Module.cuda are replaced with torch.Tensor.npu and torch.nn.Module.npu now..

    The torch.cuda.DoubleTensor is replaced with torch.npu.FloatTensor cause the double type is not supported now..

    The backend in torch.distributed.init_process_group set to hccl now..

    The torch.cuda.* and torch.cuda.amp.* are replaced with torch.npu.* and torch.npu.amp.* now..

    The device parameters have been replaced with npu in the function below:

    torch.logspace, torch.randint, torch.hann_window, torch.rand, torch.full_like, torch.ones_like, torch.rand_like, torch.randperm, torch.arange, torch.frombuffer, torch.normal, torch._empty_per_channel_affine_quantized, torch.empty_strided, torch.empty_like, torch.scalar_tensor, torch.tril_indices, torch.bartlett_window, torch.ones, torch.sparse_coo_tensor, torch.randn, torch.kaiser_window, torch.tensor, torch.triu_indices, torch.as_tensor, torch.zeros, torch.randint_like, torch.full, torch.eye, torch._sparse_csr_tensor_unsafe, torch.empty, torch._sparse_coo_tensor_unsafe, torch.blackman_window, torch.zeros_like, torch.range, torch.sparse_csr_tensor, torch.randn_like, torch.from_file, torch._cudnn_init_dropout_state, torch._empty_affine_quantized, torch.linspace, torch.hamming_window, torch.empty_quantized, torch._pin_memory, torch.autocast, torch.load, torch.Generator, torch.set_default_device, torch.Tensor.new_empty, torch.Tensor.new_empty_strided, torch.Tensor.new_full, torch.Tensor.new_ones, torch.Tensor.new_tensor, torch.Tensor.new_zeros, torch.Tensor.to, torch.nn.Module.to, torch.nn.Module.to_empty

    *************************************************************************************************************

    

  warnings.warn(msg, ImportWarning)

/usr/local/python3.9.2/lib/python3.9/site-packages/torch_npu/contrib/transfer_to_npu.py:260: RuntimeWarning: torch.jit.script and torch.jit.script_method will be disabled by transfer_to_npu, which currently does not support them, if you need to enable them, please do not use transfer_to_npu.

  warnings.warn(msg, RuntimeWarning)

Traceback (most recent call last):

  File "/app/yololv9/detect.py", line 420, in <module>

    main(opt)

  File "/app/yololv9/detect.py", line 415, in main

    run(**vars(opt))

  File "/usr/local/python3.9.2/lib/python3.9/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context

    return func(*args, **kwargs)

  File "/app/yololv9/detect.py", line 109, in run

    pred = model(im, augment=augment, visualize=visualize)

  File "/usr/local/python3.9.2/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl

    return self._call_impl(*args, **kwargs)

  File "/usr/local/python3.9.2/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl

    return forward_call(*args, **kwargs)

  File "/app/yololv9/models/common.py", line 1064, in forward

    y = self.model(im, augment=augment, visualize=visualize) if augment or visualize else self.model(im)

  File "/usr/local/python3.9.2/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl

    return self._call_impl(*args, **kwargs)

  File "/usr/local/python3.9.2/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl

    return forward_call(*args, **kwargs)

  File "/app/yololv9/models/yolo.py", line 633, in forward

    return self._forward_once(x, profile, visualize)  # single-scale inference, train

  File "/app/yololv9/models/yolo.py", line 533, in _forward_once

    x = m(x)  # run

  File "/usr/local/python3.9.2/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl

    return self._call_impl(*args, **kwargs)

  File "/usr/local/python3.9.2/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl

    return forward_call(*args, **kwargs)

  File "/app/yololv9/models/common.py", line 54, in forward

    return self.act(self.bn(self.conv(x)))

  File "/usr/local/python3.9.2/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl

    return self._call_impl(*args, **kwargs)

  File "/usr/local/python3.9.2/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl

    return forward_call(*args, **kwargs)

  File "/usr/local/python3.9.2/lib/python3.9/site-packages/torch/nn/modules/batchnorm.py", line 171, in forward

    return F.batch_norm(

  File "/usr/local/python3.9.2/lib/python3.9/site-packages/torch/nn/functional.py", line 2478, in batch_norm

    return torch.batch_norm(

RuntimeError: call aclnnBatchNorm failed, detail:EZ9999: Inner Error!

EZ9999: [PID: 4625] 2026-01-05-15:48:57.306.215 Parse dynamic kernel config fail.

        TraceBack (most recent call last):

       AclOpKernelInit failed opType

       BNInferAiCore ADD_TO_LAUNCHER_LIST_AICORE failed.

[ERROR] 2026-01-05-15:48:57 (PID:4625, Device:0, RankID:-1) ERR01100 OPS call acl api failed

root@localhost:/app/yololv9# 

本帖最后由 匿名用户2026/01/06 10:59:45 编辑

我要发帖子