如何解决华为昇腾A800双机集群部署的推理速度比单机慢？
处理中
发表于 1 小时前11查看
想请教一下，就是部署Qwen3-235B的华为升腾服务器，华为推理部署的，然后集群跑的推理速度没有单机跑的快，然后总的带宽是12.1GBps，已经关闭量化、降低 attentionOptimization和设置 maxInputTokenLen。还是没有用，不知道怎么解决？
硬件要求
部署Qwen3-235B-A22B模型进行推理至少需要2台Atlas 800I A2（8*64G）服务器
权重
权重下载
BF16原始权重下载
推理前置准备
检查机器网络情况
# 1.检查物理链接 for i in {0..7}; do hccn_tool -i $i -lldp -g | grep Ifname; done
# 2.检查链接情况 for i in {0..7}; do hccn_tool -i $i -link -g ; done
# 3.检查网络健康情况 for i in {0..7}; do hccn_tool -i $i -net_health -g ; done
# 4.查看侦测ip的配置是否正确 for i in {0..7}; do hccn_tool -i $i -netdetect -g ; done
# 5.查看网关是否配置正确 for i in {0..7}; do hccn_tool -i $i -gateway -g ; done
# 6.检查NPU底层tls校验行为一致性，建议统一全部设置为0，避免hccl报错 for i in {0..7}; do hccn_tool -i $i -tls -g ; done | grep switch
# 7.NPU底层tls校验行为置0操作，建议统一全部设置为0，避免hccl报错 for i in {0..7};do hccn_tool -i $i -tls -s enable 0;done
获取每张卡的ip地址
for i in {0..7};do hccn_tool -i $i -ip -g; done
需要用户自行创建rank_table_file.json，参考如下格式配置
{
"server_count": "2",
"server_list": [
{
"device": [
{
"device_id": "0",
"device_ip": "100.97.0.9",
"rank_id": "0"
},
{
"device_id": "1",
"device_ip": "100.97.0.10",
"rank_id": "1"
},
{
"device_id": "2",
"device_ip": "100.97.0.11",
"rank_id": "2"
},
{
"device_id": "3",
"device_ip": "100.97.0.12",
"rank_id": "3"
},
{
"device_id": "4",
"device_ip": "100.97.0.13",
"rank_id": "4"
},
{
"device_id": "5",
"device_ip": "100.97.0.14",
"rank_id": "5"
},
{
"device_id": "6",
"device_ip": "100.97.0.15",
"rank_id": "6"
},
{
"device_id": "7",
"device_ip": "100.97.0.16",
"rank_id": "7"
}
],
"server_id": "...",
"container_ip": "10.207.232.2"
},
{
"device": [
{
"device_id": "0",
"device_ip": "100.97.0.17",
"rank_id": "8"
},
{
"device_id": "1",
"device_ip": "100.97.0.18",
"rank_id": "9"
},
{
"device_id": "2",
"device_ip": "100.97.0.19",
"rank_id": "10"
},
{
"device_id": "3",
"device_ip": "100.97.0.20",
"rank_id": "11"
},
{
"device_id": "4",
"device_ip": "100.97.0.21",
"rank_id": "12"
},
{
"device_id": "5",
"device_ip": "100.97.0.22",
"rank_id": "13"
},
{
"device_id": "6",
"device_ip": "100.97.0.23",
"rank_id": "14"
},
{
"device_id": "7",
"device_ip": "100.97.0.24",
"rank_id": "15"
}
],
"server_id": "...",
"container_ip": "10.207.232.3"
}
],
"status": "completed",
"version": "1.0"
}
rank_table_file.json配置完成后，需要执行命令修改权限为640
chmod -R 640 {rank_table_file.json路径}
修改模型文件夹属组为1001 -HwHiAiUser属组（容器为Root权限可忽视），执行权限为750：
chown -R 1001:1001 {/path-to-weights/Qwen3-235B-A22B} chmod -R 750 {/path-to-weights/Qwen3-235B-A22B}
容器启动
启动容器
执行以下命令启动容器（参考）：
主机1：
docker run -itd --privileged --name=qwen3-master --net=host \
--shm-size 500g \
--device=/dev/davinci0 \
--device=/dev/davinci1 \
--device=/dev/davinci2 \
--device=/dev/davinci3 \
--device=/dev/davinci4 \
--device=/dev/davinci5 \
--device=/dev/davinci6 \
--device=/dev/davinci7 \
--device=/dev/davinci_manager \
--device=/dev/hisi_hdc \
--device /dev/devmm_svm \
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \
-v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \
-v /usr/local/sbin:/usr/local/sbin \
-v /etc/hccn.conf:/etc/hccn.conf \
-v /mnt/data/Qwen3_235/model/Qwen/Qwen3-235B-A22B:/model-weights \
-v /mnt/data/tvi3/rank_table_file.json:/tvi3/rank_table_file.json \
-it swr.cn-south-1.myhuaweicloud.com/ascendhub/mindie:2.1.RC1-800I-A2-py311-openeuler24.03-lts bash
docker exec -it qwen3-master bash
主机2：
docker run -itd --privileged --name=qwen3-slave --net=host \
--shm-size 500g \
--device=/dev/davinci0 \
--device=/dev/davinci1 \
--device=/dev/davinci2 \
--device=/dev/davinci3 \
--device=/dev/davinci4 \
--device=/dev/davinci5 \
--device=/dev/davinci6 \
--device=/dev/davinci7 \
--device=/dev/davinci_manager \
--device=/dev/hisi_hdc \
--device /dev/devmm_svm \
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \
-v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \
-v /usr/local/sbin:/usr/local/sbin \
-v /etc/hccn.conf:/etc/hccn.conf \
-v /mnt/data/Qwen3_235/model/Qwen/Qwen3-235B-A22B:/model-weights \
-v /mnt/data/tvi4/rank_table_file.json:/tvi4/rank_table_file.json \
-it swr.cn-south-1.myhuaweicloud.com/ascendhub/mindie:2.1.RC1-800I-A2-py311-openeuler24.03-lts bash
docker exec -it qwen3-slave bash
vim /usr/local/Ascend/mindie/latest/mindie-service/conf/config.json
[root@deepseek-12 model-weights]# cat /usr/local/Ascend/mindie/latest/mindie-service/conf/config.json
{
"Version" : "1.0.0",
"ServerConfig" :
{
"ipAddress" : "10.207.232.2",
"managementIpAddress" : "10.207.232.2",
"port" : 1025,
"managementPort" : 1026,
"metricsPort" : 1027,
"allowAllZeroIpListening" : false,
"maxLinkNum" : 1000,
"httpsEnabled" : false,
"fullTextEnabled" : false,
"tlsCaPath" : "security/ca/",
"tlsCaFile" : ["ca.pem"],
"tlsCert" : "security/certs/server.pem",
"tlsPk" : "security/keys/server.key.pem",
"tlsPkPwd" : "security/pass/key_pwd.txt",
"tlsCrlPath" : "security/certs/",
"tlsCrlFiles" : ["server_crl.pem"],
"managementTlsCaFile" : ["management_ca.pem"],
"managementTlsCert" : "security/certs/management/server.pem",
"managementTlsPk" : "security/keys/management/server.key.pem",
"managementTlsPkPwd" : "security/pass/management/key_pwd.txt",
"managementTlsCrlPath" : "security/management/certs/",
"managementTlsCrlFiles" : ["server_crl.pem"],
"kmcKsfMaster" : "tools/pmt/master/ksfa",
"kmcKsfStandby" : "tools/pmt/standby/ksfb",
"inferMode" : "standard",
"interCommTLSEnabled" : false,
"interCommPort" : 1121,
"interCommTlsCaPath" : "security/grpc/ca/",
"interCommTlsCaFiles" : ["ca.pem"],
"interCommTlsCert" : "security/grpc/certs/server.pem",
"interCommPk" : "security/grpc/keys/server.key.pem",
"interCommPkPwd" : "security/grpc/pass/key_pwd.txt",
"interCommTlsCrlPath" : "security/grpc/certs/",
"interCommTlsCrlFiles" : ["server_crl.pem"],
"openAiSupport" : "vllm",
"tokenTimeout" : 600,
"e2eTimeout" : 600,
"distDPServerEnabled":false
},
"BackendConfig" : {
"backendName" : "mindieservice_llm_engine",
"modelInstanceNumber" : 1,
"npuDeviceIds" : [[0,1,2,3,4,5,6,7]],
"tokenizerProcessNumber" : 8,
"multiNodesInferEnabled" : true,
"multiNodesInferPort" : 1120,
"interNodeTLSEnabled" : false,
"interNodeTlsCaPath" : "security/grpc/ca/",
"interNodeTlsCaFiles" : ["ca.pem"],
"interNodeTlsCert" : "security/grpc/certs/server.pem",
"interNodeTlsPk" : "security/grpc/keys/server.key.pem",
"interNodeTlsPkPwd" : "security/grpc/pass/mindie_server_key_pwd.txt",
"interNodeTlsCrlPath" : "security/grpc/certs/",
"interNodeTlsCrlFiles" : ["server_crl.pem"],
"interNodeKmcKsfMaster" : "tools/pmt/master/ksfa",
"interNodeKmcKsfStandby" : "tools/pmt/standby/ksfb",
"ModelDeployConfig" :
{
"maxSeqLen" : 40960,
"maxInputTokenLen" : 40959,
"truncation" : false,
"ModelConfig" : [
{
"modelInstanceType" : "Standard",
"modelName" : "Qwen3",
"modelWeightPath" : "/model-weights",
"worldSize" : 8,
"cpuMemSize" : 5,
"npuMemSize" : -1,
"backendType" : "atb",
"trustRemoteCode" : false,
"async_scheduler_wait_time": 120,
"kv_trans_timeout": 10,
"kv_link_timeout": 1080
}
]
},
"ScheduleConfig" :
{
"templateType" : "Standard",
"templateName" : "Standard_LLM",
"cacheBlockSize" : 128,
"maxPrefillBatchSize" : 64,
"maxPrefillTokens" : 40960,
"prefillTimeMsPerReq" : 150,
"prefillPolicyType" : 0,
"decodeTimeMsPerReq" : 50,
"decodePolicyType" : 0,
"maxBatchSize" : 256,
"maxIterTimes" : 40959,
"maxPreemptCount" : 0,
"supportSelectBatch" : false,
"maxQueueDelayMicroseconds" : 5000
}
}
}
两个节点都要执行设置基础环境变量
source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh source /usr/local/Ascend/atb-models/set_env.sh source /usr/local/Ascend/mindie/set_env.sh
两个节点都要执行开启通信环境变量
export ATB_LLM_HCCL_ENABLE=1
export ATB_LLM_COMM_BACKEND="hccl"
export HCCL_CONNECT_TIMEOUT=7200
export HCCL_EXEC_TIMEOUT=0
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export WORLD_SIZE=16
主节点：
export MIES_CONTAINER_IP=10.207.232.2
export RANK_TABLE_FILE=/tvi3/rank_table_file.json
export OMP_NUM_THREADS=1
export NPU_MEMORY_FRACTION=0.95
子节点：
export MIES_CONTAINER_IP=10.207.232.3
export RANK_TABLE_FILE=/tvi4/rank_table_file.json
export OMP_NUM_THREADS=1
export NPU_MEMORY_FRACTION=0.95
# 拉起服务化 cd /usr/local/Ascend/mindie/latest/mindie-service/ ./bin/mindieservice_daemon
点赞
收藏
回复
分享
举报
已解决
如何解决华为昇腾A800双机集群部署的推理速度比单机慢？
处理中
发表于 1 小时前11查看
想请教一下，就是部署Qwen3-235B的华为升腾服务器，华为推理部署的，然后集群跑的推理速度没有单机跑的快，然后总的带宽是12.1GBps，已经关闭量化、降低 attentionOptimization和设置 maxInputTokenLen。还是没有用，不知道怎么解决？
硬件要求
部署Qwen3-235B-A22B模型进行推理至少需要2台Atlas 800I A2（8*64G）服务器
权重
权重下载
BF16原始权重下载
推理前置准备
检查机器网络情况
# 1.检查物理链接 for i in {0..7}; do hccn_tool -i $i -lldp -g | grep Ifname; done
# 2.检查链接情况 for i in {0..7}; do hccn_tool -i $i -link -g ; done
# 3.检查网络健康情况 for i in {0..7}; do hccn_tool -i $i -net_health -g ; done
# 4.查看侦测ip的配置是否正确 for i in {0..7}; do hccn_tool -i $i -netdetect -g ; done
# 5.查看网关是否配置正确 for i in {0..7}; do hccn_tool -i $i -gateway -g ; done
# 6.检查NPU底层tls校验行为一致性，建议统一全部设置为0，避免hccl报错 for i in {0..7}; do hccn_tool -i $i -tls -g ; done | grep switch
# 7.NPU底层tls校验行为置0操作，建议统一全部设置为0，避免hccl报错 for i in {0..7};do hccn_tool -i $i -tls -s enable 0;done
获取每张卡的ip地址
for i in {0..7};do hccn_tool -i $i -ip -g; done
需要用户自行创建rank_table_file.json，参考如下格式配置
{
"server_count": "2",
"server_list": [
{
"device": [
{
"device_id": "0",
"device_ip": "100.97.0.9",
"rank_id": "0"
},
{
"device_id": "1",
"device_ip": "100.97.0.10",
"rank_id": "1"
},
{
"device_id": "2",
"device_ip": "100.97.0.11",
"rank_id": "2"
},
{
"device_id": "3",
"device_ip": "100.97.0.12",
"rank_id": "3"
},
{
"device_id": "4",
"device_ip": "100.97.0.13",
"rank_id": "4"
},
{
"device_id": "5",
"device_ip": "100.97.0.14",
"rank_id": "5"
},
{
"device_id": "6",
"device_ip": "100.97.0.15",
"rank_id": "6"
},
{
"device_id": "7",
"device_ip": "100.97.0.16",
"rank_id": "7"
}
],
"server_id": "...",
"container_ip": "10.207.232.2"
},
{
"device": [
{
"device_id": "0",
"device_ip": "100.97.0.17",
"rank_id": "8"
},
{
"device_id": "1",
"device_ip": "100.97.0.18",
"rank_id": "9"
},
{
"device_id": "2",
"device_ip": "100.97.0.19",
"rank_id": "10"
},
{
"device_id": "3",
"device_ip": "100.97.0.20",
"rank_id": "11"
},
{
"device_id": "4",
"device_ip": "100.97.0.21",
"rank_id": "12"
},
{
"device_id": "5",
"device_ip": "100.97.0.22",
"rank_id": "13"
},
{
"device_id": "6",
"device_ip": "100.97.0.23",
"rank_id": "14"
},
{
"device_id": "7",
"device_ip": "100.97.0.24",
"rank_id": "15"
}
],
"server_id": "...",
"container_ip": "10.207.232.3"
}
],
"status": "completed",
"version": "1.0"
}
参数
说明
server_count
总节点数
rank_table_file.json配置完成后，需要执行命令修改权限为640
chmod -R 640 {rank_table_file.json路径}
修改模型文件夹属组为1001 -HwHiAiUser属组（容器为Root权限可忽视），执行权限为750：
chown -R 1001:1001 {/path-to-weights/Qwen3-235B-A22B} chmod -R 750 {/path-to-weights/Qwen3-235B-A22B}
容器启动
启动容器
执行以下命令启动容器（参考）：
主机1：
docker run -itd --privileged --name=qwen3-master --net=host \
--shm-size 500g \
--device=/dev/davinci0 \
--device=/dev/davinci1 \
--device=/dev/davinci2 \
--device=/dev/davinci3 \
--device=/dev/davinci4 \
--device=/dev/davinci5 \
--device=/dev/davinci6 \
--device=/dev/davinci7 \
--device=/dev/davinci_manager \
--device=/dev/hisi_hdc \
--device /dev/devmm_svm \
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \
-v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \
-v /usr/local/sbin:/usr/local/sbin \
-v /etc/hccn.conf:/etc/hccn.conf \
-v /mnt/data/Qwen3_235/model/Qwen/Qwen3-235B-A22B:/model-weights \
-v /mnt/data/tvi3/rank_table_file.json:/tvi3/rank_table_file.json \
-it swr.cn-south-1.myhuaweicloud.com/ascendhub/mindie:2.1.RC1-800I-A2-py311-openeuler24.03-lts bash
docker exec -it qwen3-master bash
主机2：
docker run -itd --privileged --name=qwen3-slave --net=host \
--shm-size 500g \
--device=/dev/davinci0 \
--device=/dev/davinci1 \
--device=/dev/davinci2 \
--device=/dev/davinci3 \
--device=/dev/davinci4 \
--device=/dev/davinci5 \
--device=/dev/davinci6 \
--device=/dev/davinci7 \
--device=/dev/davinci_manager \
--device=/dev/hisi_hdc \
--device /dev/devmm_svm \
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \
-v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \
-v /usr/local/sbin:/usr/local/sbin \
-v /etc/hccn.conf:/etc/hccn.conf \
-v /mnt/data/Qwen3_235/model/Qwen/Qwen3-235B-A22B:/model-weights \
-v /mnt/data/tvi4/rank_table_file.json:/tvi4/rank_table_file.json \
-it swr.cn-south-1.myhuaweicloud.com/ascendhub/mindie:2.1.RC1-800I-A2-py311-openeuler24.03-lts bash
docker exec -it qwen3-slave bash
vim /usr/local/Ascend/mindie/latest/mindie-service/conf/config.json
[root@deepseek-12 model-weights]# cat /usr/local/Ascend/mindie/latest/mindie-service/conf/config.json
{
"Version" : "1.0.0",
"ServerConfig" :
{
"ipAddress" : "10.207.232.2",
"managementIpAddress" : "10.207.232.2",
"port" : 1025,
"managementPort" : 1026,
"metricsPort" : 1027,
"allowAllZeroIpListening" : false,
"maxLinkNum" : 1000,
"httpsEnabled" : false,
"fullTextEnabled" : false,
"tlsCaPath" : "security/ca/",
"tlsCaFile" : ["ca.pem"],
"tlsCert" : "security/certs/server.pem",
"tlsPk" : "security/keys/server.key.pem",
"tlsPkPwd" : "security/pass/key_pwd.txt",
"tlsCrlPath" : "security/certs/",
"tlsCrlFiles" : ["server_crl.pem"],
"managementTlsCaFile" : ["management_ca.pem"],
"managementTlsCert" : "security/certs/management/server.pem",
"managementTlsPk" : "security/keys/management/server.key.pem",
"managementTlsPkPwd" : "security/pass/management/key_pwd.txt",
"managementTlsCrlPath" : "security/management/certs/",
"managementTlsCrlFiles" : ["server_crl.pem"],
"kmcKsfMaster" : "tools/pmt/master/ksfa",
"kmcKsfStandby" : "tools/pmt/standby/ksfb",
"inferMode" : "standard",
"interCommTLSEnabled" : false,
"interCommPort" : 1121,
"interCommTlsCaPath" : "security/grpc/ca/",
"interCommTlsCaFiles" : ["ca.pem"],
"interCommTlsCert" : "security/grpc/certs/server.pem",
"interCommPk" : "security/grpc/keys/server.key.pem",
"interCommPkPwd" : "security/grpc/pass/key_pwd.txt",
"interCommTlsCrlPath" : "security/grpc/certs/",
"interCommTlsCrlFiles" : ["server_crl.pem"],
"openAiSupport" : "vllm",
"tokenTimeout" : 600,
"e2eTimeout" : 600,
"distDPServerEnabled":false
},
"BackendConfig" : {
"backendName" : "mindieservice_llm_engine",
"modelInstanceNumber" : 1,
"npuDeviceIds" : [[0,1,2,3,4,5,6,7]],
"tokenizerProcessNumber" : 8,
"multiNodesInferEnabled" : true,
"multiNodesInferPort" : 1120,
"interNodeTLSEnabled" : false,
"interNodeTlsCaPath" : "security/grpc/ca/",
"interNodeTlsCaFiles" : ["ca.pem"],
"interNodeTlsCert" : "security/grpc/certs/server.pem",
"interNodeTlsPk" : "security/grpc/keys/server.key.pem",
"interNodeTlsPkPwd" : "security/grpc/pass/mindie_server_key_pwd.txt",
"interNodeTlsCrlPath" : "security/grpc/certs/",
"interNodeTlsCrlFiles" : ["server_crl.pem"],
"interNodeKmcKsfMaster" : "tools/pmt/master/ksfa",
"interNodeKmcKsfStandby" : "tools/pmt/standby/ksfb",
"ModelDeployConfig" :
{
"maxSeqLen" : 40960,
"maxInputTokenLen" : 40959,
"truncation" : false,
"ModelConfig" : [
{
"modelInstanceType" : "Standard",
"modelName" : "Qwen3",
"modelWeightPath" : "/model-weights",
"worldSize" : 8,
"cpuMemSize" : 5,
"npuMemSize" : -1,
"backendType" : "atb",
"trustRemoteCode" : false,
"async_scheduler_wait_time": 120,
"kv_trans_timeout": 10,
"kv_link_timeout": 1080
}
]
},
"ScheduleConfig" :
{
"templateType" : "Standard",
"templateName" : "Standard_LLM",
"cacheBlockSize" : 128,
"maxPrefillBatchSize" : 64,
"maxPrefillTokens" : 40960,
"prefillTimeMsPerReq" : 150,
"prefillPolicyType" : 0,
"decodeTimeMsPerReq" : 50,
"decodePolicyType" : 0,
"maxBatchSize" : 256,
"maxIterTimes" : 40959,
"maxPreemptCount" : 0,
"supportSelectBatch" : false,
"maxQueueDelayMicroseconds" : 5000
}
}
}
两个节点都要执行设置基础环境变量
source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh source /usr/local/Ascend/atb-models/set_env.sh source /usr/local/Ascend/mindie/set_env.sh
两个节点都要执行开启通信环境变量
export ATB_LLM_HCCL_ENABLE=1
export ATB_LLM_COMM_BACKEND="hccl"
export HCCL_CONNECT_TIMEOUT=7200
export HCCL_EXEC_TIMEOUT=0
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export WORLD_SIZE=16
主节点：
export MIES_CONTAINER_IP=10.207.232.2
export RANK_TABLE_FILE=/tvi3/rank_table_file.json
export OMP_NUM_THREADS=1
export NPU_MEMORY_FRACTION=0.95
子节点：
export MIES_CONTAINER_IP=10.207.232.3
export RANK_TABLE_FILE=/tvi4/rank_table_file.json
export OMP_NUM_THREADS=1
export NPU_MEMORY_FRACTION=0.95
# 拉起服务化 cd /usr/local/Ascend/mindie/latest/mindie-service/ ./bin/mindieservice_daemon