Qwen3.5-27B × Ascend 910A × Transformers 适配踩坑总结

docker pull swr.cn-south-1.myhuaweicloud.com/ascendhub/cann：8.3.rc1-910-ubuntu22.04-py3.11

docker run -itd --privileged \
    --network host \
    --name cann_8.3.rc1 \
    --device /dev/davinci1 \
    --device /dev/davinci_manager \
    --device /dev/devmm_svm \
    --device /dev/hisi_hdc \
    -v /usr/local/dcmi:/usr/local/dcmi \
    -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
    -v /etc/ascend_install.info:/etc/ascend_install.info \
    -v /mnt/sdc:/mnt/sdc \
swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:8.3.rc1-910-ubuntu22.04-py3.11 bash

docker exec -it cann_8.3.rc1 bash

# 下载软件包
wget https://download.pytorch.org/whl/cpu/torch-2.8.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl
# 安装命令
pip3 install torch-2.8.0+cpu-cp311-cp311-manylinux_2_28_aarch64.whl

# 下载插件包
wget https://gitcode.com/Ascend/pytorch/releases/download/v7.2.0-pytorch2.8.0/torch_npu-2.8.0-cp311-cp311-manylinux_2_28_aarch64.whl
# 安装命令
pip3 install torch_npu-2.8.0-cp311-cp311-manylinux_2_28_aarch64.whl

# 更新transformers库
pip install transformers==5.11.0

import time
import threading

import torch
import torch_npu

from transformers import (
    AutoProcessor,
    Qwen3_5ForConditionalGeneration,
    TextIteratorStreamer,
)

# ====================================
# Ascend配置
# ====================================

torch.npu.set_compile_mode(jit_compile=False)

MODEL_PATH = "/mnt/sdc/modelscope/Qwen3.5-27B/"

print("Loading processor...")

processor = AutoProcessor.from_pretrained(
    MODEL_PATH,
    trust_remote_code=True
)

tokenizer = processor.tokenizer

print("Loading model...")

model = Qwen3_5ForConditionalGeneration.from_pretrained(
    MODEL_PATH,
    trust_remote_code=True,
    torch_dtype=torch.float16,
    device_map="auto",
    low_cpu_mem_usage=True,
    attn_implementation="eager",
)

model.eval()

print("Model loaded.")


# ====================================
# 输入
# ====================================

messages = [
    {"role": "system", "content": "你是一个乐于助人的助手。"},
    {"role": "user", "content": "介绍一下你自己。"}
]

prompt = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True
)

inputs = tokenizer(
    prompt,
    return_tensors="pt"
)

device = next(model.parameters()).device

inputs = {
    k: v.to(device)
    for k, v in inputs.items()
}

# ====================================
# Streamer
# ====================================

streamer = TextIteratorStreamer(
    tokenizer,
    skip_prompt=True,
    skip_special_tokens=True
)

generate_kwargs = dict(
    **inputs,

    streamer=streamer,

    max_new_tokens=512,

    do_sample=False,      # Ascend避开multinomial

    num_beams=1,

    use_cache=True,

    eos_token_id=tokenizer.eos_token_id,
    pad_token_id=tokenizer.eos_token_id,
)

print("\nAssistant:\n")

start_time = time.time()
first_token_time = None

generated_text = ""

thread = threading.Thread(
    target=model.generate,
    kwargs=generate_kwargs
)

thread.start()

for text in streamer:

    if first_token_time is None:
        first_token_time = time.time()

    print(text, end="", flush=True)

    generated_text += text

thread.join()

end_time = time.time()

print("\n")

# ====================================
# TPS统计
# ====================================

output_tokens = len(
    tokenizer.encode(
        generated_text,
        add_special_tokens=False
    )
)

total_time = end_time - start_time

if first_token_time:
    ttft = first_token_time - start_time
else:
    ttft = total_time

gen_time = max(end_time - first_token_time, 1e-6)

tps = output_tokens / gen_time

print("====================================")
print(f"Output Tokens : {output_tokens}")
print(f"TTFT          : {ttft:.3f} sec")
print(f"Generation    : {gen_time:.3f} sec")
print(f"Tokens/sec    : {tps:.2f}")
print("====================================")

python qwen3.5_inference.py

# ❌ 错误
from transformers import AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained(MODEL_PATH, ...)

# ✅ 正确
from transformers import Qwen3_5ForConditionalGeneration
model = Qwen3_5ForConditionalGeneration.from_pretrained(MODEL_PATH, ...)

RuntimeError: "uniform_" not implemented for 'BFloat16' on NPU

# ❌ 错误
model = ...from_pretrained(..., torch_dtype=torch.bfloat16)

# ✅ 正确：强制使用 float16
model = ...from_pretrained(..., torch_dtype=torch.float16)

RuntimeError: tensor shape mismatch (24 vs 4)

model = ...from_pretrained(
    ...,
    attn_implementation="eager",   # 放弃 sdpa，回退到标准实现
)

RuntimeError: MultinomialWithReplacement is not supported on this device

outputs = model.generate(
    **inputs,
    do_sample=False,    # 关闭随机采样，改用贪心解码
    num_beams=1,        # 配合贪心，避免 beam search 相关算子
    ...
)