本机环境介绍，硬件 g2280 4卡 300 i pro，os:5.10.0-216.0.0.115.oe2203sp4.aarch64 docker: 版本 26.1.14 docker 镜像 :1.0.0-300I-Duo-py311-openeuler24.03-lts
自查环境：
1.容器内 npu驱动正常
[root@hwgpu1 log]# npu-smi info
+--------------------------------------------------------------------------------------------------------+
| npu-smi 24.1.rc2 Version: 24.1.rc2 |
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) |
| Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) |
+===============================+=================+======================================================+
| 0 310P3 | OK | NA 40 10 / 10 |
| 0 0 | 0000:01:00.0 | 0 1858 / 21527 |
+===============================+=================+======================================================+
| 32 310P3 | OK | NA 40 0 / 0 |
| 0 1 | 0000:02:00.0 | 0 1806 / 21527 |
+===============================+=================+======================================================+
| 32768 310P3 | OK | NA 41 0 / 0 |
| 0 2 | 0000:81:00.0 | 0 1808 / 21527 |
+===============================+=================+======================================================+
| 32800 310P3 | OK | NA 43 0 / 0 |
| 0 3 | 0000:82:00.0 | 0 1809 / 21527 |
+===============================+=================+======================================================+
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Chip | Process id | Process name | Process memory(MB) |
+===============================+=================+======================================================+
2.容器内 mindie相关组件正常
[root@hwgpu1 Ascend]# pip list | grep mindie
mindie_llm 1.0.0
mindiebenchmark 1.0.0
mindieclient 1.0.0
mindiesd 1.0.0
mindietorch 1.0.0+torch2.1.0.abi0
[root@hwgpu1 Ascend]# ll
total 32
drwxr-xr-x. 1 root root 4096 Jan 14 16:42 ascend-toolkit
drwxr-xr-x. 1 root root 4096 Jan 14 16:47 atb-models
drwxr-xr-x. 12 root root 4096 Mar 7 10:58 driver
drwxr-x---. 1 root root 4096 Jan 14 16:47 mindie
drwxr-xr-x. 3 root root 4096 Jan 14 16:42 nnal
按照官方帖子 https://www.hiascend.com/software/modelzoo/models/detail/7af7f0ddab754ccd9dfe31dacf56b149 qwq32不需要转换权重，
mindiesevice config.json 配置 ，"npuDeviceIds" : [[0,1,2,3]], "worldSize" : 4,
启动容器后 ./mindieservice_daemon 报错 看这几行报错貌似和hcc有关系，但是官方的镜像应该不会有问题吧？ 求助管理
[2025-03-10 20:02:43,217] [321313] [281467021214048] [llm] [INFO] [flash_causal_qwen2.py-394] : <<<<<<<after transdata k_caches[0].shape=torch.Size([1, 16, 128, 16])
[2025-03-10 20:02:43.865437] [error] [321858] [hccl_runner.cpp:132] AllGatherHcclRunner:0 HcclCommInitRootInfo fail, error:2, rank:0, rankSize:4
[2025-03-10 20:02:43.865491] [error] [321858] [comm_pool.h:43] CommPool commCreateFunc fail
[2025-03-10 20:02:43.865500] [error] [321858] [hccl_runner.cpp:72] AllGatherHcclRunner:0 get hccl comm fail by rank:0
[2025-03-10 20:02:43.871095] [error] [322755] [all_gather_hccl_runner.cpp:38] hcclComm is null, rank: 0
[2025-03-10 20:02:43.871125] [error] [322755] [runner.cpp:115] AllGatherHcclRunner_1_1:1 Execute Failed. st: 3
[2025-03-10 20:02:43.871137] [error] [322755] [graph_runner.cpp:906] WordEmbeddingRunner_1:0 node[1] execute fail, runner name:AllGatherHcclRunner
[2025-03-10 20:02:43.871148] [error] [322755] [runner.cpp:115] WordEmbeddingRunner_1:1 Execute Failed. st: 3
[2025-03-10 20:02:43.871155] [error] [322755] [operation_base.cpp:726] WordEmbedding_1 execute WordEmbeddingRunner fail
[2025-03-10 20:02:43.871162] [error] [322755] [operation_base.cpp:764] WordEmbedding_1 execute fail, error code: 3
2025-03-10 20:02:43,871 [ERROR] model.py:39 - [Model] >>> Exception:Execute fail, enable log: export ASDOPS_LOG_LEVEL=ERROR, export ASDOPS_LOG_TO_STDOUT=1 to findthe first error. For more details, see the MindIE official document.
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/model_wrapper/model.py", line 37, in initialize
return self.python_model.initialize(config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/model_wrapper/standard_model.py", line 146, in initialize
self.generator = Generator(
^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 119, in __init__
self.warm_up(max_prefill_tokens, max_seq_len, max_input_len, max_iter_times, inference_mode)
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 303, in warm_up
raise e
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 296, in warm_up
self._generate_inputs_warm_up_backend(input_metadata, inference_mode, dummy=True)
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 378, in _generate_inputs_warm_up_backend
self.generator_backend.warm_up(model_inputs, inference_mode=inference_mode)
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 198, in warm_up
super().warm_up(model_inputs)
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_backend.py", line 170, in warm_up
_ = self.forward(model_inputs, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/mindie_llm/utils/decorators/time_decorator.py", line 38, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 153, in forward
logits = self.model_wrapper.forward(model_inputs, self.cache_pool.npu_cache, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 89, in forward
logits = self.forward_tensor(
^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 116, in forward_tensor
logits = self.model_runner.forward(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/runner/model_runner.py", line 193, in forward
return self.model.forward(**kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/models/base/flash_causal_lm.py", line 458, in forward
logits = self.execute_ascend_operator(acl_inputs, acl_param, is_prefill)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2/flash_causal_qwen2.py", line 374, in execute_ascend_operator
acl_model_out = model_operation.execute(acl_inputs, acl_param)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: Execute fail, enable log: export ASDOPS_LOG_LEVEL=ERROR, export ASDOPS_LOG_TO_STDOUT=1 to findthe first error. For more details, see the MindIE official document.
点赞
收藏
回复
分享
举报
已解决
本机环境介绍，硬件 g2280 4卡 300 i pro，os:5.10.0-216.0.0.115.oe2203sp4.aarch64 docker: 版本 26.1.14 docker 镜像 :1.0.0-300I-Duo-py311-openeuler24.03-lts
自查环境：
1.容器内 npu驱动正常
[root@hwgpu1 log]# npu-smi info
+--------------------------------------------------------------------------------------------------------+
| npu-smi 24.1.rc2 Version: 24.1.rc2 |
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) |
| Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) |
+===============================+=================+======================================================+
| 0 310P3 | OK | NA 40 10 / 10 |
| 0 0 | 0000:01:00.0 | 0 1858 / 21527 |
+===============================+=================+======================================================+
| 32 310P3 | OK | NA 40 0 / 0 |
| 0 1 | 0000:02:00.0 | 0 1806 / 21527 |
+===============================+=================+======================================================+
| 32768 310P3 | OK | NA 41 0 / 0 |
| 0 2 | 0000:81:00.0 | 0 1808 / 21527 |
+===============================+=================+======================================================+
| 32800 310P3 | OK | NA 43 0 / 0 |
| 0 3 | 0000:82:00.0 | 0 1809 / 21527 |
+===============================+=================+======================================================+
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Chip | Process id | Process name | Process memory(MB) |
+===============================+=================+======================================================+
2.容器内 mindie相关组件正常
[root@hwgpu1 Ascend]# pip list | grep mindie
mindie_llm 1.0.0
mindiebenchmark 1.0.0
mindieclient 1.0.0
mindiesd 1.0.0
mindietorch 1.0.0+torch2.1.0.abi0
[root@hwgpu1 Ascend]# ll
total 32
drwxr-xr-x. 1 root root 4096 Jan 14 16:42 ascend-toolkit
drwxr-xr-x. 1 root root 4096 Jan 14 16:47 atb-models
drwxr-xr-x. 12 root root 4096 Mar 7 10:58 driver
drwxr-x---. 1 root root 4096 Jan 14 16:47 mindie
drwxr-xr-x. 3 root root 4096 Jan 14 16:42 nnal
按照官方帖子 https://www.hiascend.com/software/modelzoo/models/detail/7af7f0ddab754ccd9dfe31dacf56b149 qwq32不需要转换权重，
mindiesevice config.json 配置 ，"npuDeviceIds" : [[0,1,2,3]], "worldSize" : 4,
启动容器后 ./mindieservice_daemon 报错 看这几行报错貌似和hcc有关系，但是官方的镜像应该不会有问题吧？ 求助管理
[2025-03-10 20:02:43,217] [321313] [281467021214048] [llm] [INFO] [flash_causal_qwen2.py-394] : <<<<<<<after transdata k_caches[0].shape=torch.Size([1, 16, 128, 16])
[2025-03-10 20:02:43.865437] [error] [321858] [hccl_runner.cpp:132] AllGatherHcclRunner:0 HcclCommInitRootInfo fail, error:2, rank:0, rankSize:4
[2025-03-10 20:02:43.865491] [error] [321858] [comm_pool.h:43] CommPool commCreateFunc fail
[2025-03-10 20:02:43.865500] [error] [321858] [hccl_runner.cpp:72] AllGatherHcclRunner:0 get hccl comm fail by rank:0
[2025-03-10 20:02:43.871095] [error] [322755] [all_gather_hccl_runner.cpp:38] hcclComm is null, rank: 0
[2025-03-10 20:02:43.871125] [error] [322755] [runner.cpp:115] AllGatherHcclRunner_1_1:1 Execute Failed. st: 3
[2025-03-10 20:02:43.871137] [error] [322755] [graph_runner.cpp:906] WordEmbeddingRunner_1:0 node[1] execute fail, runner name:AllGatherHcclRunner
[2025-03-10 20:02:43.871148] [error] [322755] [runner.cpp:115] WordEmbeddingRunner_1:1 Execute Failed. st: 3
[2025-03-10 20:02:43.871155] [error] [322755] [operation_base.cpp:726] WordEmbedding_1 execute WordEmbeddingRunner fail
[2025-03-10 20:02:43.871162] [error] [322755] [operation_base.cpp:764] WordEmbedding_1 execute fail, error code: 3
2025-03-10 20:02:43,871 [ERROR] model.py:39 - [Model] >>> Exception:Execute fail, enable log: export ASDOPS_LOG_LEVEL=ERROR, export ASDOPS_LOG_TO_STDOUT=1 to findthe first error. For more details, see the MindIE official document.
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/model_wrapper/model.py", line 37, in initialize
return self.python_model.initialize(config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/model_wrapper/standard_model.py", line 146, in initialize
self.generator = Generator(
^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 119, in __init__
self.warm_up(max_prefill_tokens, max_seq_len, max_input_len, max_iter_times, inference_mode)
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 303, in warm_up
raise e
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 296, in warm_up
self._generate_inputs_warm_up_backend(input_metadata, inference_mode, dummy=True)
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 378, in _generate_inputs_warm_up_backend
self.generator_backend.warm_up(model_inputs, inference_mode=inference_mode)
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 198, in warm_up
super().warm_up(model_inputs)
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_backend.py", line 170, in warm_up
_ = self.forward(model_inputs, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/mindie_llm/utils/decorators/time_decorator.py", line 38, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 153, in forward
logits = self.model_wrapper.forward(model_inputs, self.cache_pool.npu_cache, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 89, in forward
logits = self.forward_tensor(
^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 116, in forward_tensor
logits = self.model_runner.forward(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/runner/model_runner.py", line 193, in forward
return self.model.forward(**kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/models/base/flash_causal_lm.py", line 458, in forward
logits = self.execute_ascend_operator(acl_inputs, acl_param, is_prefill)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2/flash_causal_qwen2.py", line 374, in execute_ascend_operator
acl_model_out = model_operation.execute(acl_inputs, acl_param)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: Execute fail, enable log: export ASDOPS_LOG_LEVEL=ERROR, export ASDOPS_LOG_TO_STDOUT=1 to findthe first error. For more details, see the MindIE official document.