ERROR 04-09 08:00:00 engine.py:400] call aclnnSwiGlu failed, detail:EZ1001: [PID: 21197] 2025-04-09-08:00:00.622.630 Get path and read binary_info_config.json failed, please check if the opp_kernel package is installed! ERROR 04-09 08:00:00 engine.py:400] TraceBack (most recent call last): ERROR 04-09 08:00:00 engine.py:400] Check NnopbaseCollecterWork(binCollecter.get()) failed ERROR 04-09 08:00:00 engine.py:400] Assert ((NnopbaseInit()) == 0) failed ERROR 04-09 08:00:00 engine.py:400] Check NnopbaseCreateExecutorSpace(&executorSpace) failed ERROR 04-09 08:00:00 engine.py:400] ERROR 04-09 08:00:00 engine.py:400] [ERROR] 2025-04-09-08:00:00 (PID:21197, Device:0, RankID:-1) ERR01100 OPS call acl api failed ERROR 04-09 08:00:00 engine.py:400] Traceback (most recent call last): ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/engine/multiprocessing/engine.py", line 391, in run_mp_engine ERROR 04-09 08:00:00 engine.py:400] engine = MQLLMEngine.from_engine_args(engine_args=engine_args, ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/engine/multiprocessing/engine.py", line 124, in from_engine_args ERROR 04-09 08:00:00 engine.py:400] return cls(ipc_path=ipc_path, ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/engine/multiprocessing/engine.py", line 76, in init ERROR 04-09 08:00:00 engine.py:400] self.engine = LLMEngine(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 276, in init ERROR 04-09 08:00:00 engine.py:400] self._initialize_kv_caches() ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 421, in _initialize_kv_caches ERROR 04-09 08:00:00 engine.py:400] self.model_executor.determine_num_available_blocks()) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/executor/executor_base.py", line 102, in determine_num_available_blocks ERROR 04-09 08:00:00 engine.py:400] results = self.collective_rpc("determine_num_available_blocks") ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/executor/executor_base.py", line 316, in collective_rpc ERROR 04-09 08:00:00 engine.py:400] return self._run_workers(method, *args, **(kwargs or {})) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/executor/mp_distributed_executor.py", line 185, in _run_workers ERROR 04-09 08:00:00 engine.py:400] driver_worker_output = run_method(self.driver_worker, sent_method, ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/utils.py", line 2196, in run_method ERROR 04-09 08:00:00 engine.py:400] return func(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context ERROR 04-09 08:00:00 engine.py:400] return func(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm_ascend/worker/worker.py", line 227, in determine_num_available_blocks ERROR 04-09 08:00:00 engine.py:400] self.model_runner.profile_run() ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context ERROR 04-09 08:00:00 engine.py:400] return func(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm_ascend/worker/model_runner.py", line 1366, in profile_run ERROR 04-09 08:00:00 engine.py:400] self.execute_model(model_input, kv_caches, intermediate_tensors) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context ERROR 04-09 08:00:00 engine.py:400] return func(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm_ascend/worker/model_runner.py", line 1146, in execute_model ERROR 04-09 08:00:00 engine.py:400] hidden_or_intermediate_states = model_executable( ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl ERROR 04-09 08:00:00 engine.py:400] return self._call_impl(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl ERROR 04-09 08:00:00 engine.py:400] return forward_call(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/model_executor/models/qwen2_5_vl.py", line 1068, in forward ERROR 04-09 08:00:00 engine.py:400] hidden_states = self.language_model.model( ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/compilation/decorators.py", line 172, in call ERROR 04-09 08:00:00 engine.py:400] return self.forward(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py", line 348, in forward ERROR 04-09 08:00:00 engine.py:400] hidden_states, residual = layer( ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl ERROR 04-09 08:00:00 engine.py:400] return self._call_impl(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl ERROR 04-09 08:00:00 engine.py:400] return forward_call(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py", line 257, in forward ERROR 04-09 08:00:00 engine.py:400] hidden_states = self.mlp(hidden_states) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl ERROR 04-09 08:00:00 engine.py:400] return self._call_impl(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl ERROR 04-09 08:00:00 engine.py:400] return forward_call(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py", line 96, in forward ERROR 04-09 08:00:00 engine.py:400] x = self.act_fn(gate_up) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl ERROR 04-09 08:00:00 engine.py:400] return self._call_impl(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl ERROR 04-09 08:00:00 engine.py:400] return forward_call(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm/model_executor/custom_op.py", line 25, in forward ERROR 04-09 08:00:00 engine.py:400] return self._forward_method(*args, **kwargs) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/vllm_ascend/ops/activation.py", line 25, in silu_and_mul_forward_oot ERROR 04-09 08:00:00 engine.py:400] out = torch_npu.npu_swiglu(x) ERROR 04-09 08:00:00 engine.py:400] File "/usr/local/python3.10/lib/python3.10/site-packages/torch/_ops.py", line 1116, in call ERROR 04-09 08:00:00 engine.py:400] return self._op(*args, **(kwargs or {})) ERROR 04-09 08:00:00 engine.py:400] RuntimeError: call aclnnSwiGlu failed, detail:EZ1001: [PID: 21197] 2025-04-09-08:00:00.622.630 Get path and read binary_info_config.json failed, please check if the opp_kernel package is installed! ERROR 04-09 08:00:00 engine.py:400] TraceBack (most recent call last): ERROR 04-09 08:00:00 engine.py:400] Check NnopbaseCollecterWork(binCollecter.get()) failed ERROR 04-09 08:00:00 engine.py:400] Assert ((NnopbaseInit()) == 0) failed ERROR 04-09 08:00:00 engine.py:400] Check NnopbaseCreateExecutorSpace(&executorSpace) failed ERROR 04-09 08:00:00 engine.py:400] ERROR 04-09 08:00:00 engine.py:400] [ERROR] 2025-04-09-08:00:00 (PID:21197, Device:0, RankID:-1) ERR01100 OPS call acl api failed