昇腾社区首页
支持
消息
我的论坛主页
论坛首页
/
为什么单310P3卡vllm部署Qwen3-14B-W8A8成功，但是用四张310P3部署就失败
收藏回复举报
为什么单310P3卡vllm部署Qwen3-14B-W8A8成功，但是用四张310P3部署就失败
t('forum.solved') 已解决
发表于2025-10-14 09:30:49
0 查看

为什么单310P3卡vllm部署Qwen3-14B-W8A8成功，但是用四张310P3部署就失败

单卡部署成功

services:
  vllm-ascend-qwen3:
    image: quay.io/ascend/vllm-ascend:v0.9.2rc1-310p-openeuler
    container_name: vllm-ascend-qwen3
    network_mode: host
    working_dir: /workspace
    restart: unless-stopped

    environment:
      - ASCEND_VISIBLE_DEVICES=0
      - VLLM_USE_MODELSCOPE=true
      - MODEL_PATH=/workspace/Qwen3-14B-W8A8
      - MODEL_NAME=Qwen3-14B-W8A8
      - HOST_PORT=19000
      - PYTORCH_NPU_ALLOC_CONF=max_split_size_mb:256

    devices:
      - /dev/davinci0:/dev/davinci0
      - /dev/davinci_manager:/dev/davinci_manager
      - /dev/devmm_svm:/dev/devmm_svm
      - /dev/hisi_hdc:/dev/hisi_hdc

    volumes:
      # 挂载当前目录到容器的/workspace（模型文件应该在当前目录下）
      - .:/workspace
      # Ascend相关挂载
      - /usr/local/Ascend/driver:/usr/local/Ascend/driver:ro
      - /usr/local/dcmi:/usr/local/dcmi:ro
      - /usr/local/bin/npu-smi:/usr/local/bin/npu-smi:ro
      - /usr/local/sbin/:/usr/local/sbin:ro
      - /etc/ascend_install.info:/etc/ascend_install.info

    command: >
      bash -c "vllm serve /workspace/Qwen3-14B-W8A8
      --served-model-name Qwen3-14B-W8A8
      --max_model_len 8096
      --tensor-parallel-size 1
      --compilation-config '{\"custom_ops\":[\"none\", \"+rms_norm\", \"+rotary_embedding\"]}' 
      --enforce-eager 
      --dtype float16 
      --trust-remote-code 
      --port 19000 
      --gpu-memory-utilization  0.8
      --quantization ascend"

    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:19000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s

四张卡部署

services:
  vllm-ascend-qwen3:
    image: quay.io/ascend/vllm-ascend:v0.9.2rc1-310p-openeuler
    container_name: vllm-ascend-qwen3
    network_mode: host
    working_dir: /workspace
    restart: unless-stopped

    environment:
      - ASCEND_VISIBLE_DEVICES=0
      - VLLM_USE_MODELSCOPE=true
      - MODEL_PATH=/workspace/Qwen3-14B-W8A8
      - MODEL_NAME=Qwen3-14B-W8A8
      - HOST_PORT=19000
      - PYTORCH_NPU_ALLOC_CONF=max_split_size_mb:256

    devices:
      - /dev/davinci0:/dev/davinci0
      - /dev/davinci5:/dev/davinci5
      - /dev/davinci3:/dev/davinci3
      - /dev/davinci4:/dev/davinci4
      - /dev/davinci_manager:/dev/davinci_manager
      - /dev/devmm_svm:/dev/devmm_svm
      - /dev/hisi_hdc:/dev/hisi_hdc

    volumes:
      # 挂载当前目录到容器的/workspace（模型文件应该在当前目录下）
      - .:/workspace
      # Ascend相关挂载
      - /usr/local/Ascend/driver:/usr/local/Ascend/driver:ro
      - /usr/local/dcmi:/usr/local/dcmi:ro
      - /usr/local/bin/npu-smi:/usr/local/bin/npu-smi:ro
      - /usr/local/sbin/:/usr/local/sbin:ro
      - /etc/ascend_install.info:/etc/ascend_install.info

    command: >
      bash -c "vllm serve /workspace/Qwen3-14B-W8A8
      --served-model-name Qwen3-14B-W8A8
      --max_model_len 8096
      --tensor-parallel-size 4
      --compilation-config '{\"custom_ops\":[\"none\", \"+rms_norm\", \"+rotary_embedding\"]}' 
      --enforce-eager 
      --dtype float16 
      --trust-remote-code 
      --port 19000 
      --gpu-memory-utilization  0.8
      --quantization ascend"

    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:19000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s

报错内容

vllm-ascend-qwen3  | (VllmWorker rank=3 pid=893) ERROR 10-14 01:29:19 [multiproc_executor.py:522]         Failed to allocate resource[DeviceMemory] with info [size:32]. Reason: Memory resources are exhausted.
vllm-ascend-qwen3  | (VllmWorker rank=3 pid=893) ERROR 10-14 01:29:19 [multiproc_executor.py:522] 
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522] WorkerProc hit an exception.
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522] Traceback (most recent call last):
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/vllm-workspace/vllm/vllm/v1/executor/multiproc_executor.py", line 517, in worker_busy_loop
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     output = func(*args, **kwargs)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/worker_v1.py", line 152, in determine_available_memory
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     self.model_runner.profile_run()
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/model_runner_v1.py", line 1679, in profile_run
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     hidden_states = self._dummy_run(self.max_num_tokens)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/usr/local/python3.10.17/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     return func(*args, **kwargs)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/model_runner_v1.py", line 1663, in _dummy_run
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     hidden_states = model(
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/usr/local/python3.10.17/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     return self._call_impl(*args, **kwargs)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/usr/local/python3.10.17/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     return forward_call(*args, **kwargs)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/vllm-workspace/vllm/vllm/model_executor/models/qwen3.py", line 302, in forward
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     hidden_states = self.model(input_ids, positions, intermediate_tensors,
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/vllm-workspace/vllm/vllm/compilation/decorators.py", line 173, in __call__
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     return self.forward(*args, **kwargs)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/vllm-workspace/vllm/vllm/model_executor/models/qwen2.py", line 348, in forward
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     hidden_states = self.get_input_embeddings(input_ids)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/vllm-workspace/vllm/vllm/model_executor/models/qwen2.py", line 335, in get_input_embeddings
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     return self.embed_tokens(input_ids)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/usr/local/python3.10.17/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     return self._call_impl(*args, **kwargs)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/usr/local/python3.10.17/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     return forward_call(*args, **kwargs)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/vllm-workspace/vllm-ascend/vllm_ascend/ops/vocab_parallel_embedding.py", line 63, in vocab_parallel_embedding_forward
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     output = tensor_model_parallel_all_reduce(output_parallel)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/vllm-workspace/vllm/vllm/distributed/communication_op.py", line 14, in tensor_model_parallel_all_reduce
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     return get_tp_group().all_reduce(input_)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/vllm-workspace/vllm/vllm/distributed/parallel_state.py", line 359, in all_reduce
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     return self._all_reduce_out_place(input_)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/vllm-workspace/vllm/vllm/distributed/parallel_state.py", line 362, in _all_reduce_out_place
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     return self.device_communicator.all_reduce(input_)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/vllm-workspace/vllm/vllm/distributed/device_communicators/base_device_communicator.py", line 112, in all_reduce
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     dist.all_reduce(input_, group=self.device_group)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/usr/local/python3.10.17/lib/python3.10/site-packages/torch/distributed/c10d_logger.py", line 83, in wrapper
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     return func(*args, **kwargs)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]   File "/usr/local/python3.10.17/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 2501, in all_reduce
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]     work = group.allreduce([tensor], opts)
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522] RuntimeError: create_config:build/CMakeFiles/torch_npu.dir/compiler_depend.ts:148 HCCL function error: hcclCommInitRootInfoConfig(numRanks, &rootInfo, rank, config, &(comm->hcclComm_)), error code is 2
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522] [ERROR] 2025-10-14-01:29:19 (PID:630, Device:1, RankID:-1) ERR02200 DIST call hccl api failed.
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522] EL0004: [PID: 630] 2025-10-14-01:29:19.676.405 Failed to allocate memory.
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]         Possible Cause: Available memory is insufficient.
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]         Solution: Close applications not in use.
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]         TraceBack (most recent call last):
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]         rtMalloc execute failed, reason=[driver error:out of memory][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:53]
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]         Failed to allocate resource[DeviceMemory] with info [size:32]. Reason: Memory resources are exhausted.
vllm-ascend-qwen3  | (VllmWorker rank=1 pid=630) ERROR 10-14 01:29:19 [multiproc_executor.py:522]

本帖最后由 匿名用户 于 2025/10/14 14:56:20 编辑

我要发帖子

我们使用cookie来确保您的高速浏览体验。继续浏览本站，即表示您同意我们使用cookie。查看详情