使用vllm-ascend 把 reranker部署起来了


docker run \
  --name Qwen3-Reranker-4B \
  --network=host \
  --privileged \
  --device /dev/davinci_manager \
  --device /dev/devmm_svm \
  --device /dev/hisi_hdc \
  -v /usr/local/dcmi:/usr/local/dcmi \
  -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
  -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
  -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
  -v /etc/ascend_install.info:/etc/ascend_install.info \
  -v /root/.cache:/root/.cache \
  -it quay.io/ascend/vllm-ascend:v0.9.2rc1-310p-openeuler bash


export ASCEND_RT_VISIBLE_DEVICES=1


export ASCEND_RT_VISIBLE_DEVICES=1
vllm serve /root/.cache/Qwen/Qwen3-Reranker-4B \
  --host 0.0.0.0 --port 8086 \
  --tensor-parallel-size 1 \
  --max-model-len 32768 \
  --max-num-batched-tokens 32768 \
  --max-num-seqs 50 \
  --block-size 128 \
  --enable-prefix-caching \
  --trust-remote-code \
  --enforce-eager \
  --dtype float16 \
  --gpu-memory-utilization 0.6 \ 
  --served-model-name Qwen3-Reranker-4B \
  --hf_overrides '{"architectures":["Qwen3ForSequenceClassification"],"classifier_from_token":["no", "yes"],"is_original_qwen3_reranker":true}'

指定下有显存的卡，其他参数自己根据自己资源调整一下