如题
ARNING 09-16 10:54:42 [_custom_ops.py:20] Failed to import from vllm._C with ModuleNotFoundError("No module named 'vllm._C'")
(EngineCore_DP0 pid=555) INFO 09-16 10:54:45 [parallel_state.py:1165] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0, EP rank 0
(EngineCore_DP0 pid=555) INFO 09-16 10:54:45 [model_runner_v1.py:2345] Starting to load model /dubhe/data/service_TJ/Qwen2.5-32B-Instruct-GPTQ-Int8...
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] EngineCore failed to start.
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] Traceback (most recent call last):
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 709, in run_engine_core
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] engine_core = EngineCoreProc(*args, **kwargs)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 505, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] super().__init__(vllm_config, executor_class, log_stats,
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 82, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self.model_executor = executor_class(vllm_config)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/executor/executor_base.py", line 54, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self._init_executor()
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/executor/uniproc_executor.py", line 49, in _init_executor
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self.collective_rpc("load_model")
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/executor/uniproc_executor.py", line 58, in collective_rpc
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] answer = run_method(self.driver_worker, method, args, kwargs)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/utils/__init__.py", line 3060, in run_method
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] return func(*args, **kwargs)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/worker_v1.py", line 249, in load_model
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self.model_runner.load_model()
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/model_runner_v1.py", line 2348, in load_model
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self.model = get_model(vllm_config=self.vllm_config)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/model_executor/model_loader/__init__.py", line 119, in get_model
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] return loader.load_model(vllm_config=vllm_config,
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/model_executor/model_loader/base_loader.py", line 45, in load_model
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] model = initialize_model(vllm_config=vllm_config,
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/model_executor/model_loader/utils.py", line 64, in initialize_model
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] return model_class(vllm_config=vllm_config, prefix=prefix)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/model_executor/models/qwen2.py", line 469, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self.model = Qwen2Model(vllm_config=vllm_config,
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/compilation/decorators.py", line 199, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/model_executor/models/qwen2.py", line 309, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self.embed_tokens = VocabParallelEmbedding(
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm-ascend/vllm_ascend/ops/vocab_parallel_embedding.py", line 81, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] quant_method = quant_config.get_quant_method(self, prefix=prefix)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm-ascend/vllm_ascend/quantization/quant_config.py", line 115, in get_quant_method
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] if self.is_layer_skipped_ascend(prefix,
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm-ascend/vllm_ascend/quantization/quant_config.py", line 147, in is_layer_skipped_ascend
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] is_skipped = self.quant_description[prefix + '.weight'] == "FLOAT"
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] KeyError: 'model.embed_tokens.weight'
(EngineCore_DP0 pid=555) Process EngineCore_DP0:
(EngineCore_DP0 pid=555) Traceback (most recent call last):
(EngineCore_DP0 pid=555) File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/process.py", line 314, in _bootstrap
(EngineCore_DP0 pid=555) self.run()
(EngineCore_DP0 pid=555) File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/process.py", line 108, in run
(EngineCore_DP0 pid=555) self._target(*self._args, **self._kwargs)
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 722, in run_engine_core
(EngineCore_DP0 pid=555) raise e
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 709, in run_engine_core
(EngineCore_DP0 pid=555) engine_core = EngineCoreProc(*args, **kwargs)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 505, in __init__
(EngineCore_DP0 pid=555) super().__init__(vllm_config, executor_class, log_stats,
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 82, in __init__
(EngineCore_DP0 pid=555) self.model_executor = executor_class(vllm_config)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/executor/executor_base.py", line 54, in __init__
(EngineCore_DP0 pid=555) self._init_executor()
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/executor/uniproc_executor.py", line 49, in _init_executor
(EngineCore_DP0 pid=555) self.collective_rpc("load_model")
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/executor/uniproc_executor.py", line 58, in collective_rpc
(EngineCore_DP0 pid=555) answer = run_method(self.driver_worker, method, args, kwargs)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/utils/__init__.py", line 3060, in run_method
(EngineCore_DP0 pid=555) return func(*args, **kwargs)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/worker_v1.py", line 249, in load_model
(EngineCore_DP0 pid=555) self.model_runner.load_model()
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/model_runner_v1.py", line 2348, in load_model
(EngineCore_DP0 pid=555) self.model = get_model(vllm_config=self.vllm_config)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/model_executor/model_loader/__init__.py", line 119, in get_model
(EngineCore_DP0 pid=555) return loader.load_model(vllm_config=vllm_config,
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/model_executor/model_loader/base_loader.py", line 45, in load_model
(EngineCore_DP0 pid=555) model = initialize_model(vllm_config=vllm_config,
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/model_executor/model_loader/utils.py", line 64, in initialize_model
(EngineCore_DP0 pid=555) return model_class(vllm_config=vllm_config, prefix=prefix)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/model_executor/models/qwen2.py", line 469, in __init__
(EngineCore_DP0 pid=555) self.model = Qwen2Model(vllm_config=vllm_config,
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/compilation/decorators.py", line 199, in __init__
(EngineCore_DP0 pid=555) old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs)
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/model_executor/models/qwen2.py", line 309, in __init__
(EngineCore_DP0 pid=555) self.embed_tokens = VocabParallelEmbedding(
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm-ascend/vllm_ascend/ops/vocab_parallel_embedding.py", line 81, in __init__
(EngineCore_DP0 pid=555) quant_method = quant_config.get_quant_method(self, prefix=prefix)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm-ascend/vllm_ascend/quantization/quant_config.py", line 115, in get_quant_method
(EngineCore_DP0 pid=555) if self.is_layer_skipped_ascend(prefix,
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm-ascend/vllm_ascend/quantization/quant_config.py", line 147, in is_layer_skipped_ascend
(EngineCore_DP0 pid=555) is_skipped = self.quant_description[prefix + '.weight'] == "FLOAT"
(EngineCore_DP0 pid=555) ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) KeyError: 'model.embed_tokens.weight'
(APIServer pid=286) Traceback (most recent call last):
(APIServer pid=286) File "<frozen runpy>", line 198, in _run_module_as_main
(APIServer pid=286) File "<frozen runpy>", line 88, in _run_code
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/entrypoints/openai/api_server.py", line 2011, in <module>
(APIServer pid=286) uvloop.run(run_server(args))
(APIServer pid=286) File "/usr/local/python3.11.13/lib/python3.11/site-packages/uvloop/__init__.py", line 105, in run
(APIServer pid=286) return runner.run(wrapper())
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/usr/local/python3.11.13/lib/python3.11/asyncio/runners.py", line 118, in run
(APIServer pid=286) return self._loop.run_until_complete(task)
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
(APIServer pid=286) File "/usr/local/python3.11.13/lib/python3.11/site-packages/uvloop/__init__.py", line 61, in wrapper
(APIServer pid=286) return await main
(APIServer pid=286) ^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/entrypoints/openai/api_server.py", line 1941, in run_server
(APIServer pid=286) await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/entrypoints/openai/api_server.py", line 1961, in run_server_worker
(APIServer pid=286) async with build_async_engine_client(
(APIServer pid=286) File "/usr/local/python3.11.13/lib/python3.11/contextlib.py", line 210, in __aenter__
(APIServer pid=286) return await anext(self.gen)
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/entrypoints/openai/api_server.py", line 179, in build_async_engine_client
(APIServer pid=286) async with build_async_engine_client_from_engine_args(
(APIServer pid=286) File "/usr/local/python3.11.13/lib/python3.11/contextlib.py", line 210, in __aenter__
(APIServer pid=286) return await anext(self.gen)
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/entrypoints/openai/api_server.py", line 221, in build_async_engine_client_from_engine_args
(APIServer pid=286) async_llm = AsyncLLM.from_vllm_config(
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/utils/__init__.py", line 1589, in inner
(APIServer pid=286) return fn(*args, **kwargs)
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/async_llm.py", line 212, in from_vllm_config
(APIServer pid=286) return cls(
(APIServer pid=286) ^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/async_llm.py", line 136, in __init__
(APIServer pid=286) self.engine_core = EngineCoreClient.make_async_mp_client(
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/core_client.py", line 102, in make_async_mp_client
(APIServer pid=286) return AsyncMPClient(*client_args)
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/core_client.py", line 769, in __init__
(APIServer pid=286) super().__init__(
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/core_client.py", line 448, in __init__
(APIServer pid=286) with launch_core_engines(vllm_config, executor_class,
(APIServer pid=286) File "/usr/local/python3.11.13/lib/python3.11/contextlib.py", line 144, in __exit__
(APIServer pid=286) next(self.gen)
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/utils.py", line 729, in launch_core_engines
(APIServer pid=286) wait_for_engine_startup(
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/utils.py", line 782, in wait_for_engine_startup
(APIServer pid=286) raise RuntimeError("Engine core initialization failed. "
(APIServer pid=286) RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
(APIServer pid=286) [ERROR] 2025-09-16-10:54:54 (PID:286, Device:-1, RankID:-1) ERR99999 UNKNOWN applicaiton exception
点赞
收藏
回复
分享
举报
已解决
如题
ARNING 09-16 10:54:42 [_custom_ops.py:20] Failed to import from vllm._C with ModuleNotFoundError("No module named 'vllm._C'")
(EngineCore_DP0 pid=555) INFO 09-16 10:54:45 [parallel_state.py:1165] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0, EP rank 0
(EngineCore_DP0 pid=555) INFO 09-16 10:54:45 [model_runner_v1.py:2345] Starting to load model /dubhe/data/service_TJ/Qwen2.5-32B-Instruct-GPTQ-Int8...
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] EngineCore failed to start.
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] Traceback (most recent call last):
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 709, in run_engine_core
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] engine_core = EngineCoreProc(*args, **kwargs)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 505, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] super().__init__(vllm_config, executor_class, log_stats,
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 82, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self.model_executor = executor_class(vllm_config)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/executor/executor_base.py", line 54, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self._init_executor()
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/executor/uniproc_executor.py", line 49, in _init_executor
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self.collective_rpc("load_model")
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/executor/uniproc_executor.py", line 58, in collective_rpc
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] answer = run_method(self.driver_worker, method, args, kwargs)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/utils/__init__.py", line 3060, in run_method
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] return func(*args, **kwargs)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/worker_v1.py", line 249, in load_model
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self.model_runner.load_model()
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/model_runner_v1.py", line 2348, in load_model
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self.model = get_model(vllm_config=self.vllm_config)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/model_executor/model_loader/__init__.py", line 119, in get_model
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] return loader.load_model(vllm_config=vllm_config,
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/model_executor/model_loader/base_loader.py", line 45, in load_model
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] model = initialize_model(vllm_config=vllm_config,
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/model_executor/model_loader/utils.py", line 64, in initialize_model
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] return model_class(vllm_config=vllm_config, prefix=prefix)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/model_executor/models/qwen2.py", line 469, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self.model = Qwen2Model(vllm_config=vllm_config,
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/compilation/decorators.py", line 199, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm/vllm/model_executor/models/qwen2.py", line 309, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] self.embed_tokens = VocabParallelEmbedding(
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm-ascend/vllm_ascend/ops/vocab_parallel_embedding.py", line 81, in __init__
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] quant_method = quant_config.get_quant_method(self, prefix=prefix)
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm-ascend/vllm_ascend/quantization/quant_config.py", line 115, in get_quant_method
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] if self.is_layer_skipped_ascend(prefix,
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] File "/vllm-workspace/vllm-ascend/vllm_ascend/quantization/quant_config.py", line 147, in is_layer_skipped_ascend
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] is_skipped = self.quant_description[prefix + '.weight'] == "FLOAT"
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) ERROR 09-16 10:54:47 [core.py:718] KeyError: 'model.embed_tokens.weight'
(EngineCore_DP0 pid=555) Process EngineCore_DP0:
(EngineCore_DP0 pid=555) Traceback (most recent call last):
(EngineCore_DP0 pid=555) File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/process.py", line 314, in _bootstrap
(EngineCore_DP0 pid=555) self.run()
(EngineCore_DP0 pid=555) File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/process.py", line 108, in run
(EngineCore_DP0 pid=555) self._target(*self._args, **self._kwargs)
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 722, in run_engine_core
(EngineCore_DP0 pid=555) raise e
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 709, in run_engine_core
(EngineCore_DP0 pid=555) engine_core = EngineCoreProc(*args, **kwargs)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 505, in __init__
(EngineCore_DP0 pid=555) super().__init__(vllm_config, executor_class, log_stats,
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/v1/engine/core.py", line 82, in __init__
(EngineCore_DP0 pid=555) self.model_executor = executor_class(vllm_config)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/executor/executor_base.py", line 54, in __init__
(EngineCore_DP0 pid=555) self._init_executor()
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/executor/uniproc_executor.py", line 49, in _init_executor
(EngineCore_DP0 pid=555) self.collective_rpc("load_model")
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/executor/uniproc_executor.py", line 58, in collective_rpc
(EngineCore_DP0 pid=555) answer = run_method(self.driver_worker, method, args, kwargs)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/utils/__init__.py", line 3060, in run_method
(EngineCore_DP0 pid=555) return func(*args, **kwargs)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/worker_v1.py", line 249, in load_model
(EngineCore_DP0 pid=555) self.model_runner.load_model()
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/model_runner_v1.py", line 2348, in load_model
(EngineCore_DP0 pid=555) self.model = get_model(vllm_config=self.vllm_config)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/model_executor/model_loader/__init__.py", line 119, in get_model
(EngineCore_DP0 pid=555) return loader.load_model(vllm_config=vllm_config,
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/model_executor/model_loader/base_loader.py", line 45, in load_model
(EngineCore_DP0 pid=555) model = initialize_model(vllm_config=vllm_config,
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/model_executor/model_loader/utils.py", line 64, in initialize_model
(EngineCore_DP0 pid=555) return model_class(vllm_config=vllm_config, prefix=prefix)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/model_executor/models/qwen2.py", line 469, in __init__
(EngineCore_DP0 pid=555) self.model = Qwen2Model(vllm_config=vllm_config,
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/compilation/decorators.py", line 199, in __init__
(EngineCore_DP0 pid=555) old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs)
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm/vllm/model_executor/models/qwen2.py", line 309, in __init__
(EngineCore_DP0 pid=555) self.embed_tokens = VocabParallelEmbedding(
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm-ascend/vllm_ascend/ops/vocab_parallel_embedding.py", line 81, in __init__
(EngineCore_DP0 pid=555) quant_method = quant_config.get_quant_method(self, prefix=prefix)
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm-ascend/vllm_ascend/quantization/quant_config.py", line 115, in get_quant_method
(EngineCore_DP0 pid=555) if self.is_layer_skipped_ascend(prefix,
(EngineCore_DP0 pid=555) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) File "/vllm-workspace/vllm-ascend/vllm_ascend/quantization/quant_config.py", line 147, in is_layer_skipped_ascend
(EngineCore_DP0 pid=555) is_skipped = self.quant_description[prefix + '.weight'] == "FLOAT"
(EngineCore_DP0 pid=555) ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=555) KeyError: 'model.embed_tokens.weight'
(APIServer pid=286) Traceback (most recent call last):
(APIServer pid=286) File "<frozen runpy>", line 198, in _run_module_as_main
(APIServer pid=286) File "<frozen runpy>", line 88, in _run_code
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/entrypoints/openai/api_server.py", line 2011, in <module>
(APIServer pid=286) uvloop.run(run_server(args))
(APIServer pid=286) File "/usr/local/python3.11.13/lib/python3.11/site-packages/uvloop/__init__.py", line 105, in run
(APIServer pid=286) return runner.run(wrapper())
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/usr/local/python3.11.13/lib/python3.11/asyncio/runners.py", line 118, in run
(APIServer pid=286) return self._loop.run_until_complete(task)
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
(APIServer pid=286) File "/usr/local/python3.11.13/lib/python3.11/site-packages/uvloop/__init__.py", line 61, in wrapper
(APIServer pid=286) return await main
(APIServer pid=286) ^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/entrypoints/openai/api_server.py", line 1941, in run_server
(APIServer pid=286) await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/entrypoints/openai/api_server.py", line 1961, in run_server_worker
(APIServer pid=286) async with build_async_engine_client(
(APIServer pid=286) File "/usr/local/python3.11.13/lib/python3.11/contextlib.py", line 210, in __aenter__
(APIServer pid=286) return await anext(self.gen)
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/entrypoints/openai/api_server.py", line 179, in build_async_engine_client
(APIServer pid=286) async with build_async_engine_client_from_engine_args(
(APIServer pid=286) File "/usr/local/python3.11.13/lib/python3.11/contextlib.py", line 210, in __aenter__
(APIServer pid=286) return await anext(self.gen)
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/entrypoints/openai/api_server.py", line 221, in build_async_engine_client_from_engine_args
(APIServer pid=286) async_llm = AsyncLLM.from_vllm_config(
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/utils/__init__.py", line 1589, in inner
(APIServer pid=286) return fn(*args, **kwargs)
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/async_llm.py", line 212, in from_vllm_config
(APIServer pid=286) return cls(
(APIServer pid=286) ^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/async_llm.py", line 136, in __init__
(APIServer pid=286) self.engine_core = EngineCoreClient.make_async_mp_client(
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/core_client.py", line 102, in make_async_mp_client
(APIServer pid=286) return AsyncMPClient(*client_args)
(APIServer pid=286) ^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/core_client.py", line 769, in __init__
(APIServer pid=286) super().__init__(
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/core_client.py", line 448, in __init__
(APIServer pid=286) with launch_core_engines(vllm_config, executor_class,
(APIServer pid=286) File "/usr/local/python3.11.13/lib/python3.11/contextlib.py", line 144, in __exit__
(APIServer pid=286) next(self.gen)
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/utils.py", line 729, in launch_core_engines
(APIServer pid=286) wait_for_engine_startup(
(APIServer pid=286) File "/vllm-workspace/vllm/vllm/v1/engine/utils.py", line 782, in wait_for_engine_startup
(APIServer pid=286) raise RuntimeError("Engine core initialization failed. "
(APIServer pid=286) RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
(APIServer pid=286) [ERROR] 2025-09-16-10:54:54 (PID:286, Device:-1, RankID:-1) ERR99999 UNKNOWN applicaiton exception