docker run -it -d --net=host --shm-size=1g \
--privileged \
--name mindie \
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver:ro \
-v /usr/local/sbin:/usr/local/sbin:ro \
-v /path-to-weights:/path-to-weights:ro \
mindie:2.0.T3.1-800I-A2-py311-openeuler24.03-lts bash
export ASDOPS_LOG_TO_STDOUT=1 && python examples/models/qwen2_vl/run_pa.py --model_path /data/models/Qwen2-VL-7B-Instruct/ --trust_remote_code --input_image /data/models/test.png
[2025-04-09 18:50:43.836036] [error] [14502] [runner.cpp:115] PluginRunner_93_0_0:1 Execute Failed. st: 12
mki_log log dir:/root/atb/log exist
[2025-04-09 18:50:43.836561] [error] [14502] [graph_runner.cpp:906] LinearNoQuantRunner_93_0:0 node[0] execute fail, runner name:PluginRunner
[2025-04-09 18:50:43.836581] [error] [14502] [runner.cpp:115] LinearNoQuantRunner_93_0:1 Execute Failed. st: 12
[2025-04-09 18:50:43.836592] [error] [14502] [graph_runner.cpp:906] LmHeadRunner_93:0 node[0] execute fail, runner name:LinearNoQuantRunner
[2025-04-09 18:50:43.836602] [error] [14502] [runner.cpp:115] LmHeadRunner_93:1 Execute Failed. st: 12
[2025-04-09 18:50:43.836611] [error] [14502] [operation_base.cpp:726] LmHead_93 execute LmHeadRunner fail
[2025-04-09 18:50:43.836620] [error] [14502] [operation_base.cpp:764] LmHead_93 execute fail, error code: 12
[2025-04-09 18:50:43,837] [13709] [281461196081984] [llm] [ERROR] [logging.py-50] : release shared memory failed: release share memory error: [Errno 2] No such file or directory: '/psm_debb89b8'
[2025-04-09 18:50:44,446] [13709] [281461196081984] [llm] [INFO] [logging.py-331] : warmup_memory(GB): 22.40
[2025-04-09 18:50:44,446] [13709] [281461196081984] [llm] [INFO] [logging.py-331] : ---------------end warm_up---------------
[2025-04-09 18:50:44,550] [13709] [281461196081984] [llm] [INFO] [logging.py-331] : ------total req num: 1, infer start--------
Traceback (most recent call last):
File "/usr/local/Ascend/atb-models/examples/models/qwen2_vl/run_pa.py", line 349, in <module>
generate_texts, token_nums, latency = pa_runner.infer(
^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/examples/models/qwen2_vl/run_pa.py", line 153, in infer
generate_req(
File "/usr/local/Ascend/atb-models/examples/server/generate.py", line 438, in generate_req
req_finished, prefill_time = generate_token_with_clocking(model, cache_manager, batch)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/examples/server/generate.py", line 346, in generate_token_with_clocking
req_finished = generate_token(model, cache_manager, input_batch)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/examples/server/generate.py", line 278, in generate_token
logits = model.forward(
^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/runner/model_runner.py", line 220, in forward
return self.model.forward(**kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2_vl/flash_causal_qwen2_vl.py", line 203, in forward
return self.language_model.forward(inputs_embeds,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2_vl/flash_causal_qwen2_using_mrope.py", line 142, in forward
logits = self.execute_ascend_operator(acl_inputs, acl_param, is_prefill)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2/flash_causal_qwen2.py", line 434, in execute_ascend_operator
acl_model_out = model_operation.execute(acl_inputs, acl_param)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: Execute fail, enable log: export ASDOPS_LOG_LEVEL=ERROR, export ASDOPS_LOG_TO_STDOUT=1 to findthe first error. For more details, see the MindIE official document.
[ERROR] 2025-04-09-18:50:56 (PID:13709, Device:0, RankID:-1) ERR99999 UNKNOWN application exception
corrupted double-linked list
/usr/lib64/python3.11/multiprocessing/resource_tracker.py:254: UserWarning: resource_tracker: There appear to be 2 leaked shared_memory objects to clean up at shutdown
warnings.warn('resource_tracker: There appear to be %d '
Aborted (core dumped)
--privileged \
--name mindie \
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver:ro \
-v /usr/local/sbin:/usr/local/sbin:ro \
-v /path-to-weights:/path-to-weights:ro \
mindie:2.0.T3.1-800I-A2-py311-openeuler24.03-lts bash
export ASDOPS_LOG_TO_STDOUT=1 && python examples/models/qwen2_vl/run_pa.py --model_path /data/models/Qwen2-VL-7B-Instruct/ --trust_remote_code --input_image /data/models/test.png
[2025-04-09 18:50:43.836036] [error] [14502] [runner.cpp:115] PluginRunner_93_0_0:1 Execute Failed. st: 12
mki_log log dir:/root/atb/log exist
[2025-04-09 18:50:43.836561] [error] [14502] [graph_runner.cpp:906] LinearNoQuantRunner_93_0:0 node[0] execute fail, runner name:PluginRunner
[2025-04-09 18:50:43.836581] [error] [14502] [runner.cpp:115] LinearNoQuantRunner_93_0:1 Execute Failed. st: 12
[2025-04-09 18:50:43.836592] [error] [14502] [graph_runner.cpp:906] LmHeadRunner_93:0 node[0] execute fail, runner name:LinearNoQuantRunner
[2025-04-09 18:50:43.836602] [error] [14502] [runner.cpp:115] LmHeadRunner_93:1 Execute Failed. st: 12
[2025-04-09 18:50:43.836611] [error] [14502] [operation_base.cpp:726] LmHead_93 execute LmHeadRunner fail
[2025-04-09 18:50:43.836620] [error] [14502] [operation_base.cpp:764] LmHead_93 execute fail, error code: 12
[2025-04-09 18:50:43,837] [13709] [281461196081984] [llm] [ERROR] [logging.py-50] : release shared memory failed: release share memory error: [Errno 2] No such file or directory: '/psm_debb89b8'
[2025-04-09 18:50:44,446] [13709] [281461196081984] [llm] [INFO] [logging.py-331] : warmup_memory(GB): 22.40
[2025-04-09 18:50:44,446] [13709] [281461196081984] [llm] [INFO] [logging.py-331] : ---------------end warm_up---------------
[2025-04-09 18:50:44,550] [13709] [281461196081984] [llm] [INFO] [logging.py-331] : ------total req num: 1, infer start--------
Traceback (most recent call last):
File "/usr/local/Ascend/atb-models/examples/models/qwen2_vl/run_pa.py", line 349, in <module>
generate_texts, token_nums, latency = pa_runner.infer(
^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/examples/models/qwen2_vl/run_pa.py", line 153, in infer
generate_req(
File "/usr/local/Ascend/atb-models/examples/server/generate.py", line 438, in generate_req
req_finished, prefill_time = generate_token_with_clocking(model, cache_manager, batch)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/examples/server/generate.py", line 346, in generate_token_with_clocking
req_finished = generate_token(model, cache_manager, input_batch)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/examples/server/generate.py", line 278, in generate_token
logits = model.forward(
^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/runner/model_runner.py", line 220, in forward
return self.model.forward(**kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2_vl/flash_causal_qwen2_vl.py", line 203, in forward
return self.language_model.forward(inputs_embeds,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2_vl/flash_causal_qwen2_using_mrope.py", line 142, in forward
logits = self.execute_ascend_operator(acl_inputs, acl_param, is_prefill)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2/flash_causal_qwen2.py", line 434, in execute_ascend_operator
acl_model_out = model_operation.execute(acl_inputs, acl_param)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: Execute fail, enable log: export ASDOPS_LOG_LEVEL=ERROR, export ASDOPS_LOG_TO_STDOUT=1 to findthe first error. For more details, see the MindIE official document.
[ERROR] 2025-04-09-18:50:56 (PID:13709, Device:0, RankID:-1) ERR99999 UNKNOWN application exception
corrupted double-linked list
/usr/lib64/python3.11/multiprocessing/resource_tracker.py:254: UserWarning: resource_tracker: There appear to be 2 leaked shared_memory objects to clean up at shutdown
warnings.warn('resource_tracker: There appear to be %d '
Aborted (core dumped)