480 ^[[1;36m(APIServer pid=529)^[[0;0m INFO 11-13 11:18:23 [loggers.py:127] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 10.2 tokens/s,         Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.2%, Prefix cache hit rate: 0.0%
    481 ^[[1;36m(APIServer pid=529)^[[0;0m INFO 11-13 11:18:33 [loggers.py:127] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 9.8 tokens/s,         Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.2%, Prefix cache hit rate: 0.0%
    482 ^[[1;36m(APIServer pid=529)^[[0;0m INFO:     172.17.0.1:35720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
    483 ^[[1;36m(APIServer pid=529)^[[0;0m INFO 11-13 11:18:43 [loggers.py:127] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 6.5 tokens/s,         Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
    484 ^[[1;36m(APIServer pid=529)^[[0;0m INFO 11-13 11:18:53 [loggers.py:127] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s,         Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
    485 ^[[1;36m(APIServer pid=529)^[[0;0m WARNING 11-13 12:05:13 [protocol.py:93] The following fields were present in the request but ignored: {'enable_thinking'}
    486 ^[[1;36m(APIServer pid=529)^[[0;0m INFO:     168.0.17.22:37302 - "POST /v1/chat/completions HTTP/1.1" 200 OK
    487 mki_log mkdir /root/ascend/log/atb
    488 mki_log mkdir /root/ascend/log/atb
    489 mki_log mkdir /root/ascend/log/atb
    490 [rank4]:[E1113 12:05:13.948790630 compiler_depend.ts:444] SelfAttentionOperation setup failed!
    491 Exception raised from OperationSetup at build/third_party/op-plugin/op_plugin/CMakeFiles/op_plugin_atb.dir/compiler_depend.ts:203 (most recent call first):
    492 frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0xd4 (0xffff731b3ea4 in /usr/l        ocal/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)
    493 frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > cons        t&) + 0xe4 (0xffff73153e44 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)
    494 frame #2: atb::OperationSetup(atb::VariantPack, atb::Operation*, atb::Context*) + 0x254 (0xffff4fd7c0e8 in /usr/local/python3.11.13/lib/python3.11/site-packages/t        orch_npu/lib/libop_plugin_atb.so)
    495 frame #3: <unknown function> + 0x9ebec (0xffff4fd7ebec in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libop_plugin_atb.so)
    496 frame #4: <unknown function> + 0x26e6c10 (0xffff656d6c10 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
    497 frame #5: <unknown function> + 0x961a94 (0xffff63951a94 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
    498 frame #6: <unknown function> + 0x9644c0 (0xffff639544c0 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
    499 frame #7: <unknown function> + 0x96072c (0xffff6395072c in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
    500 frame #8: <unknown function> + 0xd43cc (0xffff72fb43cc in /lib64/libstdc++.so.6)
    501 frame #9: <unknown function> + 0x7fbb4 (0xffff7f36fbb4 in /lib64/libc.so.6)
    502 frame #10: <unknown function> + 0xe7bcc (0xffff7f3d7bcc in /lib64/libc.so.6)
    503 
    504 [rank3]:[E1113 12:05:13.948832850 compiler_depend.ts:444] SelfAttentionOperation setup failed!
    505 Exception raised from OperationSetup at build/third_party/op-plugin/op_plugin/CMakeFiles/op_plugin_atb.dir/compiler_depend.ts:203 (most recent call first):
    506 frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0xd4 (0xffff91763ea4 in /usr/l        ocal/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)
    507 frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > cons        t&) + 0xe4 (0xffff91703e44 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)
    508 frame #2: atb::OperationSetup(atb::VariantPack, atb::Operation*, atb::Context*) + 0x254 (0xffff6e40c0e8 in /usr/local/python3.11.13/lib/python3.11/site-packages/t        orch_npu/lib/libop_plugin_atb.so)
    509 frame #3: <unknown function> + 0x9ebec (0xffff6e40ebec in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libop_plugin_atb.so)
    510 frame #4: <unknown function> + 0x26e6c10 (0xffff83c86c10 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
    511 frame #5: <unknown function> + 0x961a94 (0xffff81f01a94 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
    512 frame #6: <unknown function> + 0x9644c0 (0xffff81f044c0 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
    513 frame #7: <unknown function> + 0x96072c (0xffff81f0072c in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
    514 frame #8: <unknown function> + 0xd43cc (0xffff915643cc in /lib64/libstdc++.so.6)
    515 frame #9: <unknown function> + 0x7fbb4 (0xffff9d91fbb4 in /lib64/libc.so.6)
    516 frame #10: <unknown function> + 0xe7bcc (0xffff9d987bcc in /lib64/libc.so.6)
    517 
    518 [rank2]:[E1113 12:05:13.948862500 compiler_depend.ts:444] SelfAttentionOperation setup failed!
    519 Exception raised from OperationSetup at build/third_party/op-plugin/op_plugin/CMakeFiles/op_plugin_atb.dir/compiler_depend.ts:203 (most recent call first):
    520 frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0xd4 (0xffffaf3d3ea4 in /usr/l        ocal/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)
    521 frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > cons        t&) + 0xe4 (0xffffaf373e44 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)
    522 frame #2: atb::OperationSetup(atb::VariantPack, atb::Operation*, atb::Context*) + 0x254 (0xffff8bf7c0e8 in /usr/local/python3.11.13/lib/python3.11/site-packages/t        orch_npu/lib/libop_plugin_atb.so)
    523 frame #3: <unknown function> + 0x9ebec (0xffff8bf7ebec in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libop_plugin_atb.so)
    524 frame #4: <unknown function> + 0x26e6c10 (0xffffa18f6c10 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
    525 frame #5: <unknown function> + 0x961a94 (0xffff9fb71a94 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
    526 frame #6: <unknown function> + 0x9644c0 (0xffff9fb744c0 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
    527 frame #7: <unknown function> + 0x96072c (0xffff9fb7072c in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
    528 frame #8: <unknown function> + 0xd43cc (0xffffaf1d43cc in /lib64/libstdc++.so.6)
    529 frame #9: <unknown function> + 0x7fbb4 (0xffffbb58fbb4 in /lib64/libc.so.6)
    530 frame #10: <unknown function> + 0xe7bcc (0xffffbb5f7bcc in /lib64/libc.so.6)
    531 
    532 [rank5]:[E1113 12:05:13.948938880 compiler_depend.ts:444] SelfAttentionOperation setup failed!
。。。。。。

## /root/ascend/log
[root@54efcbdceeba log]# cat atb/atb_6077_20251113122532.log 
[2025-11-13 12:25:32.191404] [error] [6491] [operation_base.cpp:384] SelfAttentionOperation_2 CheckIniMatch Failed! Supported Combs: 
[comb0]:inTensor0(float16,nd), inTensor1(float16,nd), inTensor2(float16,nd), inTensor3(float16,nd), inTensor4(int32,nd),outTensor0(float16,nd)
[comb1]:inTensor0(bf16,nd), inTensor1(bf16,nd), inTensor2(bf16,nd), inTensor3(bf16,nd), inTensor4(int32,nd),outTensor0(bf16,nd)
[comb2]:inTensor0(float16,nd), inTensor1(float16,nd), inTensor2(float16,nd), inTensor3(float16,fractal_nz), inTensor4(int32,nd),outTensor0(float16,nd)
[comb3]:inTensor0(bf16,nd), inTensor1(bf16,nd), inTensor2(bf16,nd), inTensor3(bf16,fractal_nz), inTensor4(int32,nd),outTensor0(bf16,nd)
[2025-11-13 12:25:32.192371] [error] [6491] [operation_base.cpp:625] SelfAttentionOperation_2 invalid param, setup check fail, error code: 13
[root@54efcbdceeba log]# cat atb/atb_6081_20251113122532.log 
[2025-11-13 12:25:32.191410] [error] [6486] [operation_base.cpp:384] SelfAttentionOperation_2 CheckIniMatch Failed! Supported Combs: 
[comb0]:inTensor0(float16,nd), inTensor1(float16,nd), inTensor2(float16,nd), inTensor3(float16,nd), inTensor4(int32,nd),outTensor0(float16,nd)
[comb1]:inTensor0(bf16,nd), inTensor1(bf16,nd), inTensor2(bf16,nd), inTensor3(bf16,nd), inTensor4(int32,nd),outTensor0(bf16,nd)
[comb2]:inTensor0(float16,nd), inTensor1(float16,nd), inTensor2(float16,nd), inTensor3(float16,fractal_nz), inTensor4(int32,nd),outTensor0(float16,nd)
[comb3]:inTensor0(bf16,nd), inTensor1(bf16,nd), inTensor2(bf16,nd), inTensor3(bf16,fractal_nz), inTensor4(int32,nd),outTensor0(bf16,nd)
[2025-11-13 12:25:32.192084] [error] [6486] [operation_base.cpp:625] SelfAttentionOperation_2 invalid param, setup check fail, error code: 13
[root@54efcbdceeba log]# cat atb/atb_806_20251113120513.log  
[2025-11-13 12:05:13.770395] [error] [1247] [operation_base.cpp:384] SelfAttentionOperation_2 CheckIniMatch Failed! Supported Combs: 
[comb0]:inTensor0(float16,nd), inTensor1(float16,nd), inTensor2(float16,nd), inTensor3(float16,nd), inTensor4(int32,nd),outTensor0(float16,nd)
[comb1]:inTensor0(bf16,nd), inTensor1(bf16,nd), inTensor2(bf16,nd), inTensor3(bf16,nd), inTensor4(int32,nd),outTensor0(bf16,nd)
[comb2]:inTensor0(float16,nd), inTensor1(float16,nd), inTensor2(float16,nd), inTensor3(float16,fractal_nz), inTensor4(int32,nd),outTensor0(float16,nd)
[comb3]:inTensor0(bf16,nd), inTensor1(bf16,nd), inTensor2(bf16,nd), inTensor3(bf16,fractal_nz), inTensor4(int32,nd),outTensor0(bf16,nd)
[2025-11-13 12:05:13.771499] [error] [1247] [operation_base.cpp:625] SelfAttentionOperation_2 invalid param, setup check fail, error code: 13


### plog
[root@54efcbdceeba log]# cat debug/plog/plog-806_20251113120513954.log 
[ERROR] APP(806,):2025-11-13-12:05:13.791.720 [log_inner.cpp:77]1247 build/CMakeFiles/torch_npu.dir/compiler_depend.ts:ExecFuncOpApi:448: "[PTA]:"Custom hand error:SelfAttentionOperation setup failed!
Exception raised from OperationSetup at build/third_party/op-plugin/op_plugin/CMakeFiles/op_plugin_atb.dir/compiler_depend.ts:203 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0xd4 (0xffff837f3ea4 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)
frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx""
[ERROR] APP(806,):2025-11-13-12:05:13.791.739 [log_inner.cpp:77]1247 build/CMakeFiles/torch_npu.dir/compiler_depend.ts:ExecFuncOpApi:452: "[PTA]:"Custom hand fail! name=SelfAttentionOperation, ret=100000""
[ERROR] APP(806,):2025-11-13-12:05:13.791.752 [log_inner.cpp:77]1247 build/CMakeFiles/torch_npu.dir/compiler_depend.ts:ReadQueue:438: "[PTA]:"---Thread---281468056039840: device = 0, write_idx = 1638, read_idx = 1328, status = 1, ret = 100000""
[ERROR] APP(806,):2025-11-13-12:05:14.051.673 [log_inner.cpp:77]806 build/CMakeFiles/torch_npu.dir/compiler_depend.ts:Enqueue:527: "[PTA]:"Inner error happened with CAN_EXIT status, detail: the current working operator name is SelfAttentionOperation""
[ERROR] APP(806,