版本:quay.io/ascend/vllm-ascend:v0.11.0rc1

环境变量：

export USE_ACLGRAPH=1
unset ASCEND_LAUNCH_BLOCKING
export VLLM_USE_FLASHINFER_MOE_FP16=1
export TASK_QUEUE_ENABLE=1
# 暂时关闭可能引起问题的优化
export VLLM_ASCEND_ENABLE_PREFETCH_MLP=0
export VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE=0
export VLLM_ASCEND_ENABLE_MLP_OPTIMIZE=0
unset HCCL_OP_EXPANSION_MODE
export VLLM_ASCEND_ENABLE_MATMUL_ALLREDUCE=0
unset VLLM_ASCEND_ENABLE_FLASHCOMM1
unset VLLM_ASCEND_FLASHCOMM2_PARALLEL_SIZE
export HCCL_BUFFSIZE=512

日志：

[[1;36m(APIServer pid=13206)^[[0;0m INFO 11-15 11:07:31 [entrypoints/launcher.py:42] Route: /v1/audio/translations, Methods: POST

^[[1;36m(APIServer pid=13206)^[[0;0m INFO 11-15 11:07:31 [entrypoints/launcher.py:42] Route: /rerank, Methods: POST

^[[1;36m(APIServer pid=13206)^[[0;0m INFO 11-15 11:07:31 [entrypoints/launcher.py:42] Route: /v1/rerank, Methods: POST

^[[1;36m(APIServer pid=13206)^[[0;0m INFO 11-15 11:07:31 [entrypoints/launcher.py:42] Route: /v2/rerank, Methods: POST

^[[1;36m(APIServer pid=13206)^[[0;0m INFO 11-15 11:07:31 [entrypoints/launcher.py:42] Route: /scale_elastic_ep, Methods: POST

^[[1;36m(APIServer pid=13206)^[[0;0m INFO 11-15 11:07:31 [entrypoints/launcher.py:42] Route: /is_scaling_elastic_ep, Methods: POST

^[[1;36m(APIServer pid=13206)^[[0;0m INFO 11-15 11:07:31 [entrypoints/launcher.py:42] Route: /invocations, Methods: POST

^[[1;36m(APIServer pid=13206)^[[0;0m INFO 11-15 11:07:31 [entrypoints/launcher.py:42] Route: /metrics, Methods: GET

^[[1;36m(APIServer pid=13206)^[[0;0m INFO:     Started server process [13206]

^[[1;36m(APIServer pid=13206)^[[0;0m INFO:     Waiting for application startup.

^[[1;36m(APIServer pid=13206)^[[0;0m INFO:     Application startup complete.

^[[1;36m(APIServer pid=13206)^[[0;0m INFO 11-15 11:07:41 [entrypoints/chat_utils.py:560] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.

^[[1;36m(EngineCore_DP0 pid=13344)^[[0;0m DEBUG 11-15 11:07:41 [v1/engine/core.py:743] EngineCore loop active.

^[[1;36m(Worker_TP0_EP0 pid=13480)^[[0;0m DEBUG 11-15 11:07:41 [model_runner_v1.py:1883] num_tokens: 11, moe_comm_type: MoECommType.NAIVE_MULTICAST

[rank1]:[E1115 11:07:41.659661870 compiler_depend.ts:444] ReshapeAndCacheOperation setup failed!

Exception raised from OperationSetup at build/third_party/op-plugin/op_plugin/CMakeFiles/op_plugin_atb.dir/compiler_depend.ts:203 (most recent call first):

frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0xd4 (0xffff8a593ea4 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)

frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0xe4 (0xffff8a533e44 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)

frame #2: atb::OperationSetup(atb::VariantPack, atb::Operation*, atb::Context*) + 0x254 (0xffff6721c0e8 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libop_plugin_atb.so)

frame #3: <unknown function> + 0x9ebec (0xffff6721ebec in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libop_plugin_atb.so)

frame #4: <unknown function> + 0x26e6c10 (0xffff7cc16c10 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)

frame #5: <unknown function> + 0x961a94 (0xffff7ae91a94 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)

frame #6: <unknown function> + 0x9644c0 (0xffff7ae944c0 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)

frame #7: <unknown function> + 0x96072c (0xffff7ae9072c in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)

frame #8: <unknown function> + 0xd29cc (0xffff8a3a29cc in /lib/aarch64-linux-gnu/libstdc++.so.6)

frame #9: <unknown function> + 0x80398 (0xffff96670398 in /lib/aarch64-linux-gnu/libc.so.6)

frame #10: <unknown function> + 0xe9e9c (0xffff966d9e9c in /lib/aarch64-linux-gnu/libc.so.6)

[rank6]:[E1115 11:07:41.659933900 compiler_depend.ts:444] ReshapeAndCacheOperation setup failed!

Exception raised from OperationSetup at build/third_party/op-plugin/op_plugin/CMakeFiles/op_plugin_atb.dir/compiler_depend.ts:203 (most recent call first):

frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0xd4 (0xffff7ecd3ea4 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)

frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0xe4 (0xffff7ec73e44 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)

frame #2: atb::OperationSetup(atb::VariantPack, atb::Operation*, atb::Context*) + 0x254 (0xffff5b91c0e8 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libop_plugin_atb.so)

frame #3: <unknown function> + 0x9ebec (0xffff5b91ebec in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libop_plugin_atb.so)

frame #4: <unknown function> + 0x26e6c10 (0xffff71356c10 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)

frame #5: <unknown function> + 0x961a94 (0xffff6f5d1a94 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)

frame #6: <unknown function> + 0x9644c0 (0xffff6f5d44c0 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)

frame #7: <unknown function> + 0x96072c (0xffff6f5d072c in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)

frame #8: <unknown function> + 0xd29cc (0xffff7eae29cc in /lib/aarch64-linux-gnu/libstdc++.so.6)

frame #9: <unknown function> + 0x80398 (0xffff8adb0398 in /lib/aarch64-linux-gnu/libc.so.6)

frame #10: <unknown function> + 0xe9e9c (0xffff8ae19e9c in /lib/aarch64-linux-gnu/libc.so.6)

[rank0]:[E1115 11:07:41.660052160 compiler_depend.ts:444] ReshapeAndCacheOperation setup failed!

Exception raised from OperationSetup at build/third_party/op-plugin/op_plugin/CMakeFiles/op_plugin_atb.dir/compiler_depend.ts:203 (most recent call first):

frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0xd4 (0xffffae233ea4 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)

frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0xe4 (0xffffae1d3e44 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)

frame #2: atb::OperationSetup(atb::VariantPack, atb::Operation*, atb::Context*) + 0x254 (0xffff8af1c0e8 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libop_plugin_atb.so)

frame #3: <unknown function> + 0x9ebec (0xffff8af1ebec in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libop_plugin_atb.so)

frame #4: <unknown function> + 0x26e6c10 (0xffffa08b6c10 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)

frame #5: <unknown function> + 0x961a94 (0xffff9eb31a94 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)

frame #6: <unknown function> + 0x9644c0 (0xffff9eb344c0 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)

frame #7: <unknown function> + 0x96072c (0xffff9eb3072c in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)

frame #8: <unknown function> + 0xd29cc (0xffffae0429cc in /lib/aarch64-linux-gnu/libstdc++.so.6)

frame #9: <unknown function> + 0x80398 (0xffffba310398 in /lib/aarch64-linux-gnu/libc.so.6)