昇腾论坛

设备: 300I-duo

驱动和固件信息

[root@localhost ~]# cat /usr/local/Ascend/firmware/version.info 
Version=7.0.0.5.242
firmware_version=1.0
package_version=23.0.rc3
compatible_version_drv=[23.0.rc2,23.0.rc2.],[23.0.rc3,23.0.rc3.]
[root@localhost ~]# cat /usr/local/Ascend/driver/version.info 
Version=23.0.rc3
ascendhal_version=7.31.13
aicpu_version=1.0
tdt_version=1.0
log_version=1.0
prof_version=2.0
dvppkernels_version=1.1
tsfw_version=1.0
Innerversion=V100R001C13SPC005B246
compatible_version=[V100R001C29],[V100R001C30],[V100R001C13]
compatible_version_fw=[6.4.0,6.4.99],[7.0.0,7.0.99]
package_version=23.0.rc3
[root@localhost ~]#

创建容器

docker run -it -d --net=host --shm-size=1g \
    --privileged \
    --name <container-name> \
    -v /usr/local/Ascend/driver:/usr/local/Ascend/driver:ro \
    -v /usr/local/sbin:/usr/local/sbin:ro \
    -v /data:/data \
    swr.cn-south-1.myhuaweicloud.com/ascendhub/mindie:1.0.0-300I-Duo-py311-openeuler24.03-lts bash

进入容器并按照文档安装vllm_npu0.6.2

启动vllm服务

vllm serve /data/.llmModelCache/modelscope/hub/Qwen/Qwen2___5-7B --host 0.0.0.0 --port 8889 --max-model-len 10240 --enforce-eager -tp 1

终端输出

WARNING 02-26 09:48:32 _custom_ops.py:18] Failed to import from vllm._C with ModuleNotFoundError("No module named 'vllm._C'")
INFO 02-26 09:48:32 importing.py:10] Triton not installed; certain GPU-related functions will not be available.
INFO 02-26 09:48:34 api_server.py:526] vLLM API server version 0.6.3.dev0+g7193774b.d20250225
INFO 02-26 09:48:34 api_server.py:527] args: Namespace(model_tag='/data/.llmModelCache/modelscope/hub/Qwen/Qwen2___5-7B', config='', host='0.0.0.0', port=8889, uvicorn_log_level='info', allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=False, enable_auto_tool_choice=False, tool_call_parser=None, model='/data/.llmModelCache/modelscope/hub/Qwen/Qwen2___5-7B', tokenizer=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=False, download_dir=None, load_format='auto', config_format='auto', dtype='auto', kv_cache_dtype='auto', quantization_param_path=None, max_model_len=10240, guided_decoding_backend='outlines', distributed_executor_backend=None, worker_use_ray=False, pipeline_parallel_size=1, tensor_parallel_size=1, max_parallel_loading_workers=None, ray_workers_use_nsight=False, block_size=128, enable_prefix_caching=False, disable_sliding_window=False, use_v2_block_manager=False, num_lookahead_slots=0, seed=0, swap_space=4, cpu_offload_gb=0, gpu_memory_utilization=0.9, num_gpu_blocks_override=None, max_num_batched_tokens=None, max_num_seqs=256, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, enforce_eager=True, max_context_len_to_capture=None, max_seq_len_to_capture=8192, disable_custom_all_reduce=False, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config=None, limit_mm_per_prompt=None, mm_processor_kwargs=None, enable_lora=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=False, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', num_scheduler_steps=1, multi_step_stream_outputs=False, scheduler_delay_factor=0.0, enable_chunked_prefill=None, speculative_model=None, speculative_model_quantization=None, num_speculative_tokens=None, speculative_draft_tensor_parallel_size=None, speculative_max_model_len=None, speculative_disable_by_batch_size=None, ngram_prompt_lookup_max=None, ngram_prompt_lookup_min=None, spec_decoding_acceptance_method='rejection_sampler', typical_acceptance_sampler_posterior_threshold=None, typical_acceptance_sampler_posterior_alpha=None, disable_logprobs_during_spec_decoding=None, model_loader_extra_config=None, ignore_patterns=[], preemption_mode=None, served_model_name=None, qlora_adapter_name_or_path=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, override_neuron_config=None, disable_log_requests=False, max_log_len=None, disable_fastapi_docs=False, dispatch_function=<function serve at 0xfffe207e4540>)
INFO 02-26 09:48:34 api_server.py:164] Multiprocessing frontend to use ipc:///tmp/fd0c9864-8673-4a11-bbd8-49bcd693e02b for IPC Path.
INFO 02-26 09:48:34 api_server.py:177] Started engine process with PID 2721010
WARNING 02-26 09:48:38 _custom_ops.py:18] Failed to import from vllm._C with ModuleNotFoundError("No module named 'vllm._C'")
INFO 02-26 09:48:38 importing.py:10] Triton not installed; certain GPU-related functions will not be available.
INFO 02-26 09:48:44 llm_engine.py:183] Initializing an LLM engine (v0.6.3.dev0+g7193774b.d20250225) with config: model='/data/.llmModelCache/modelscope/hub/Qwen/Qwen2___5-7B', speculative_config=None, tokenizer='/data/.llmModelCache/modelscope/hub/Qwen/Qwen2___5-7B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.float16, max_seq_len=10240, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=True, kv_cache_dtype=auto, quantization_param_path=None, device_config=npu, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=/data/.llmModelCache/modelscope/hub/Qwen/Qwen2___5-7B, use_v2_block_manager=False, num_scheduler_steps=1, multi_step_stream_outputs=False, enable_prefix_caching=False, use_async_output_proc=True, use_cached_outputs=True, mm_processor_kwargs=None)
INFO 02-26 09:48:44 selector.py:223] Cannot use _Backend.FLASH_ATTN backend on CPU.
INFO 02-26 09:48:44 selector.py:154] Using MindIE backend.
[WARN]operator(),compiler_depend.ts:661:Feature is not supportted and the possible cause is that driver and firmware packages do not match.
[W ProcessGroupGloo.cpp:695] Warning: Unable to resolve hostname to a (local) address. Using the loopback address as fallback. Manually set the network interface to bind to with GLOO_SOCKET_IFNAME. (function operator())
INFO 02-26 09:49:09 npu_model_runner.py:877] Starting to load model /data/.llmModelCache/modelscope/hub/Qwen/Qwen2___5-7B...
[2025-02-26 09:49:13,570] [2721010] [281457866983712] [llm] [INFO] [cpu_binding.py-212] : rank_id: 0, device_id: 0, numa_id: 0, shard_devices: [0, 1, 2, 3], cpus: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
[2025-02-26 09:49:13,572] [2721010] [281457866983712] [llm] [INFO] [cpu_binding.py-238] : process 2721010, new_affinity is [0, 1, 2, 3, 4, 5, 6, 7], cpu count 8
[2025-02-26 09:49:13,954] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : model_runner.quantize: None, model_runner.kv_quant_type: None, model_runner.fa_quant_type: None, model_runner.dtype: torch.float16
[2025-02-26 09:49:13,954] [2721010] [281457866983712] [llm] [INFO] [dist.py-77] : initialize_distributed has been Set
[2025-02-26 09:49:13,956] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : init tokenizer done: Qwen2TokenizerFast(name_or_path='/data/.llmModelCache/modelscope/hub/Qwen/Qwen2___5-7B', vocab_size=151643, model_max_length=131072, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'eos_token': '<|endoftext|>', 'pad_token': '<|endoftext|>', 'additional_special_tokens': ['<|im_start|>', '<|im_end|>', '<|object_ref_start|>', '<|object_ref_end|>', '<|box_start|>', '<|box_end|>', '<|quad_start|>', '<|quad_end|>', '<|vision_start|>', '<|vision_end|>', '<|vision_pad|>', '<|image_pad|>', '<|video_pad|>']}, clean_up_tokenization_spaces=False),  added_tokens_decoder={
        151643: AddedToken("<|endoftext|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151644: AddedToken("<|im_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151645: AddedToken("<|im_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151646: AddedToken("<|object_ref_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151647: AddedToken("<|object_ref_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151648: AddedToken("<|box_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151649: AddedToken("<|box_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151650: AddedToken("<|quad_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151651: AddedToken("<|quad_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151652: AddedToken("<|vision_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151653: AddedToken("<|vision_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151654: AddedToken("<|vision_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151655: AddedToken("<|image_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151656: AddedToken("<|video_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151657: AddedToken("<tool_call>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151658: AddedToken("</tool_call>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151659: AddedToken("<|fim_prefix|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151660: AddedToken("<|fim_middle|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151661: AddedToken("<|fim_suffix|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151662: AddedToken("<|fim_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151663: AddedToken("<|repo_name|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151664: AddedToken("<|file_sep|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
}
[2025-02-26 09:49:13,959] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : NPUSocInfo(soc_name='', soc_version=202, need_nz=True, matmul_nd_nz=False)
[2025-02-26 09:49:14,029] [2721010] [281457866983712] [llm] [INFO] [flash_causal_qwen2.py-115] : >>>> qwen_QwenDecoderModel is called.
[2025-02-26 09:49:22,028] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : model:
 FlashQwen2ForCausalLM(
  (rotary_embedding): PositionRotaryEmbedding()
  (attn_mask): AttentionMask()
  (transformer): FlashQwenModel(
    (wte): TensorParallelEmbedding()
    (h): ModuleList(
      (0-27): 28 x FlashQwenLayer(
        (attn): FlashQwenAttention(
          (rotary_emb): PositionRotaryEmbedding()
          (c_attn): TensorParallelColumnLinear(
            (linear): FastLinear()
          )
          (c_proj): TensorParallelRowLinear(
            (linear): FastLinear()
          )
        )
        (mlp): QwenMLP(
          (act): SiLU()
          (w2_w1): TensorParallelColumnLinear(
            (linear): FastLinear()
          )
          (c_proj): TensorParallelRowLinear(
            (linear): FastLinear()
          )
        )
        (ln_1): QwenRMSNorm()
        (ln_2): QwenRMSNorm()
      )
    )
    (ln_f): QwenRMSNorm()
  )
  (lm_head): TensorParallelHead(
    (linear): FastLinear()
  )
)
INFO 02-26 09:49:22 npu_model_runner.py:891] Loading model weights took 14.2409 GB
[2025-02-26 09:49:22,437] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,437] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,439] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,439] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,440] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,440] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,442] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,442] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,443] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,443] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,444] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,444] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,445] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,445] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,445] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,446] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,446] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,447] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,447] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,447] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,448] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,448] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,449] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,449] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,450] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,450] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,450] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,451] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,451] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,452] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,452] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,453] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,453] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,454] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,454] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,454] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,455] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,455] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,455] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,456] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,456] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,457] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,457] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,458] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,458] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,459] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,459] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,460] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,460] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,460] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,461] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,461] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,462] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,462] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,463] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,463] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,464] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,464] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,465] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,465] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,465] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,466] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,466] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,467] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,467] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,468] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,468] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,468] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,469] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,469] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,470] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,470] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,471] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,471] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,471] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,472] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,472] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,473] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,473] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,473] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,474] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,474] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,475] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,475] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,476] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,476] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,476] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,477] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,477] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,478] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,478] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,478] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,479] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,479] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,480] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,480] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,480] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,481] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,481] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,482] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,482] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,483] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,483] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,483] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,484] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,484] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,485] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,485] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,485] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,486] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,486] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,487] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:22,492] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : trans to 29
[2025-02-26 09:49:23,120] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : <<<<<<< ori k_caches[0].shape=torch.Size([256, 32, 128, 16])
[2025-02-26 09:49:23,122] [2721010] [281457866983712] [llm] [INFO] [flash_causal_qwen2.py-394] : <<<<<<<after transdata k_caches[0].shape=torch.Size([256, 32, 128, 16])
[2025-02-26 09:49:23,122] [2721010] [281457866983712] [llm] [INFO] [logging.py-331] : >>>>>>id of kcache is 281449947606640 id of vcache is 281449947606736
INFO 02-26 09:49:30 api_server.py:230] vLLM to use /tmp/tmpbe7jcs3k as PROMETHEUS_MULTIPROC_DIR
WARNING 02-26 09:49:30 serving_embedding.py:189] embedding_mode is False. Embedding API will not work.
INFO 02-26 09:49:30 launcher.py:19] Available routes are:
INFO 02-26 09:49:30 launcher.py:27] Route: /openapi.json, Methods: HEAD, GET
INFO 02-26 09:49:30 launcher.py:27] Route: /docs, Methods: HEAD, GET
INFO 02-26 09:49:30 launcher.py:27] Route: /docs/oauth2-redirect, Methods: HEAD, GET
INFO 02-26 09:49:30 launcher.py:27] Route: /redoc, Methods: HEAD, GET
INFO 02-26 09:49:30 launcher.py:27] Route: /health, Methods: GET
INFO 02-26 09:49:30 launcher.py:27] Route: /tokenize, Methods: POST
INFO 02-26 09:49:30 launcher.py:27] Route: /detokenize, Methods: POST
INFO 02-26 09:49:30 launcher.py:27] Route: /v1/models, Methods: GET
INFO 02-26 09:49:30 launcher.py:27] Route: /version, Methods: GET
INFO 02-26 09:49:30 launcher.py:27] Route: /v1/chat/completions, Methods: POST
INFO 02-26 09:49:30 launcher.py:27] Route: /v1/completions, Methods: POST
INFO 02-26 09:49:30 launcher.py:27] Route: /v1/embeddings, Methods: POST
INFO:     Started server process [2720802]
INFO:     Waiting for application startup.
INFO:     Application startup complete.
INFO:     Uvicorn running on http://0.0.0.0:8889 (Press CTRL+C to quit)

启动测试

单并发

benchmark --DatasetPath /data/et_llm/dataset/GSM8K.jsonl --DatasetType gsm8k --ModelName qwen2.5_7b --ModelPath /data/.llmModelCache/modelscope/hub/Qwen/Qwen2___5-7B --TestType vllm_client --Concurrency 1 --Tokenizer True --Http http://127.0.0.1:8889 --TestAccuracy True --MaxOutputLen 512

单并发显存及利用率 true

测试结果 true

并发数 = 10

benchmark --DatasetPath /data/et_llm/dataset/GSM8K.jsonl --DatasetType gsm8k --ModelName qwen2.5_7b --ModelPath /data/.llmModelCache/modelscope/hub/Qwen/Qwen2___5-7B --TestType vllm_client --Concurrency 10 --Tokenizer True --Http http://127.0.0.1:8889 --TestAccuracy True --MaxOutputLen 512

显存及利用率

true

测试结果 true

64并发

测试命令

benchmark --DatasetPath /data/et_llm/dataset/GSM8K.jsonl --DatasetType gsm8k --ModelName qwen2.5_7b --ModelPath /data/.llmModelCache/modelscope/hub/Qwen/Qwen2___5-7B --TestType vllm_client --Concurrency 64 --Tokenizer True --Http http://127.0.0.1:8889 --TestAccuracy True --MaxOutputLen 512

kv缓存利用情况

true

显存及利用率

true

测试结果

true

速度比较慢，单次推理基本也是10秒左右，这是正常的吗？

启动vllm服务

启动测试

单并发

并发数 = 10

关于昇腾

新闻与活动

交流与资讯

支持与服务

开源社区