使用环境:2台Atlas 800I A2 每台各八张910B3型卡
使用镜像: 2.0.T9.B020-800I-A2-py3.11-openeuler24.03-lts-aarch64
在使用两台机器部署服务化部署deepseek-R1时,在拉起服务化的过程中(./bin/mindieservice_daemon)遇到"npuDeviceID does not allow repetitive element"报错
两台机器纯模型性能测试可以跑通,但拉服务化启动时就会报错
完整日志为:
Start to parse ranktable file
Finished parsing ranktable file.
Update worldSize and npuDeviceIds of backend config successfully for Multi Nodes Inference.
npuDeviceID does not allow repetitive element
Killed
config.json已严格按照官方文档https://www.hiascend.com/software/modelzoo/models/detail/68457b8a51324310aad9a0f55c3e56e3进行配置(手动修改和整个config.json内容复制都试过了,每个config.json有关 npudeviceID 的内容都为
故应该不是config.json内容的问题)
两台机器的config.json内容相同,都为
{
"Version" : "1.0.0",
"LogConfig" : {
"logLevel" : "Info",
"logFileSize" : 20,
"logFileNum" : 20,
"logPath" : "logs/mindie-server.log"
},
"ServerConfig" : {
"ipAddress" : "10.199.117.23",
"managementIpAddress" : "10.199.117.23",
"port" : 1025,
"managementPort" : 1026,
"metricsPort" : 1027,
"allowAllZeroIpListening" : false,
"maxLinkNum" : 500,
"httpsEnabled" : false,
"fullTextEnabled" : false,
"tlsCaPath" : "security/ca/",
"tlsCaFile" : ["ca.pem"],
"tlsCert" : "security/certs/server.pem",
"tlsPk" : "security/keys/server.key.pem",
"tlsPkPwd" : "security/pass/key_pwd.txt",
"tlsCrlPath" : "security/certs/",
"tlsCrlFiles" : ["server_crl.pem"],
"managementTlsCaFile" : ["management_ca.pem"],
"managementTlsCert" : "security/certs/management/server.pem",
"managementTlsPk" : "security/keys/management/server.key.pem",
"managementTlsPkPwd" : "security/pass/management/key_pwd.txt",
"managementTlsCrlPath" : "security/management/certs/",
"managementTlsCrlFiles" : ["server_crl.pem"],
"kmcKsfMaster" : "tools/pmt/master/ksfa",
"kmcKsfStandby" : "tools/pmt/standby/ksfb",
"inferMode" : "standard",
"interCommTLSEnabled" : false,
"interCommPort" : 1121,
"interCommTlsCaPath" : "security/grpc/ca/",
"interCommTlsCaFiles" : ["ca.pem"],
"interCommTlsCert" : "security/grpc/certs/server.pem",
"interCommPk" : "security/grpc/keys/server.key.pem",
"interCommPkPwd" : "security/grpc/pass/key_pwd.txt",
"interCommTlsCrlPath" : "security/grpc/certs/",
"interCommTlsCrlFiles" : ["server_crl.pem"],
"openAiSupport" : "vllm",
"tokenTimeout":600,
"e2eTimeout":600
},
"BackendConfig" : {
"backendName" : "mindieservice_llm_engine",
"modelInstanceNumber" : 1,
"npuDeviceIds" : [[0,1,2,3,4,5,6,7]],
"tokenizerProcessNumber" : 8,
"multiNodesInferEnabled" : true,
"multiNodesInferPort" : 1120,
"interNodeTLSEnabled" : false,
"interNodeTlsCaPath" : "security/grpc/ca/",
"interNodeTlsCaFiles" : ["ca.pem"],
"interNodeTlsCert" : "security/grpc/certs/server.pem",
"interNodeTlsPk" : "security/grpc/keys/server.key.pem",
"interNodeTlsPkPwd" : "security/grpc/pass/mindie_server_key_pwd.txt",
"interNodeTlsCrlPath" : "security/grpc/certs/",
"interNodeTlsCrlFiles" : ["server_crl.pem"],
"interNodeKmcKsfMaster" : "tools/pmt/master/ksfa",
"interNodeKmcKsfStandby" : "tools/pmt/standby/ksfb",
"ModelDeployConfig" : {
"maxSeqLen" : 32768,
"maxInputTokenLen" : 32768,
"truncation" : true,
"ModelConfig" : [
{
"modelInstanceType" : "Standard",
"modelName" : "deepseek-r1",
"modelWeightPath" : "/workspace/weights/",
"worldSize" : 8,
"cpuMemSize" : 5,
"npuMemSize" : -1,
"backendType" : "atb",
"trustRemoteCode" : false
}
]
},
"ScheduleConfig" : {
"templateType" : "Standard",
"templateName" : "Standard_LLM",
"cacheBlockSize" : 128,
"maxPrefillBatchSize" : 8,
"maxPrefillTokens" : 32768,
"prefillTimeMsPerReq" : 150,
"prefillPolicyType" : 0,
"decodeTimeMsPerReq" : 50,
"decodePolicyType" : 0,
"maxBatchSize" : 8,
"maxIterTimes" : 1024,
"maxPreemptCount" : 0,
"supportSelectBatch" : false,
"maxQueueDelayMicroseconds" : 5000
}
}
}
两台机器的rank_table_file.json内容如下:
{
"version": "1.0",
"server_count": "2",
"server_list": [
{
"server_id": "10.199.117.23",
"container_ip": "10.199.117.23",
"device": [
{"device_id": "0","device_ip": "10.20.0.10","rank_id": "0"},
{"device_id": "1","device_ip": "10.20.0.11","rank_id": "1"},
{"device_id": "2","device_ip": "10.20.0.12","rank_id": "2"},
{"device_id": "3","device_ip": "10.20.0.13","rank_id": "3"},
{"device_id": "4","device_ip": "10.20.0.14","rank_id": "4"},
{"device_id": "5","device_ip": "10.20.0.15","rank_id": "5"},
{"device_id": "6","device_ip": "10.20.0.16","rank_id": "6"},
{"device_id": "7","device_ip": "10.20.0.17","rank_id": "7"}
]
},
{
"server_id": "10.199.117.25",
"container_ip": "10.199.117.25",
"device": [
{"device_id": "0","device_ip": "10.20.0.20","rank_id": "8"},
{"device_id": "1","device_ip": "10.20.0.21","rank_id": "9"},
{"device_id": "2","device_ip": "10.20.0.22","rank_id": "10"},
{"device_id": "3","device_ip": "10.20.0.23","rank_id": "11"},
{"device_id": "4","device_ip": "10.20.0.24","rank_id": "12"},
{"device_id": "5","device_ip": "10.20.0.25","rank_id": "13"},
{"device_id": "6","device_ip": "10.20.0.26","rank_id": "14"},
{"device_id": "7","device_ip": "10.20.0.27","rank_id": "15"}
]
}
],
"status": "completed"
}
目前论坛有关帖子都没有可行解决方案,需求比较急,希望能够沟通有效解决方案。
使用环境:2台Atlas 800I A2 每台各八张910B3型卡
使用镜像: 2.0.T9.B020-800I-A2-py3.11-openeuler24.03-lts-aarch64
在使用两台机器部署服务化部署deepseek-R1时,在拉起服务化的过程中(./bin/mindieservice_daemon)遇到"npuDeviceID does not allow repetitive element"报错
两台机器纯模型性能测试可以跑通,但拉服务化启动时就会报错
完整日志为:
Start to parse ranktable file
Finished parsing ranktable file.
Update worldSize and npuDeviceIds of backend config successfully for Multi Nodes Inference.
npuDeviceID does not allow repetitive element
Killed
config.json已严格按照官方文档https://www.hiascend.com/software/modelzoo/models/detail/68457b8a51324310aad9a0f55c3e56e3进行配置(手动修改和整个config.json内容复制都试过了,每个config.json有关 npudeviceID 的内容都为
故应该不是config.json内容的问题)
两台机器的config.json内容相同,都为
{
"Version" : "1.0.0",
"LogConfig" : {
"logLevel" : "Info",
"logFileSize" : 20,
"logFileNum" : 20,
"logPath" : "logs/mindie-server.log"
},
"ServerConfig" : {
"ipAddress" : "10.199.117.23",
"managementIpAddress" : "10.199.117.23",
"port" : 1025,
"managementPort" : 1026,
"metricsPort" : 1027,
"allowAllZeroIpListening" : false,
"maxLinkNum" : 500,
"httpsEnabled" : false,
"fullTextEnabled" : false,
"tlsCaPath" : "security/ca/",
"tlsCaFile" : ["ca.pem"],
"tlsCert" : "security/certs/server.pem",
"tlsPk" : "security/keys/server.key.pem",
"tlsPkPwd" : "security/pass/key_pwd.txt",
"tlsCrlPath" : "security/certs/",
"tlsCrlFiles" : ["server_crl.pem"],
"managementTlsCaFile" : ["management_ca.pem"],
"managementTlsCert" : "security/certs/management/server.pem",
"managementTlsPk" : "security/keys/management/server.key.pem",
"managementTlsPkPwd" : "security/pass/management/key_pwd.txt",
"managementTlsCrlPath" : "security/management/certs/",
"managementTlsCrlFiles" : ["server_crl.pem"],
"kmcKsfMaster" : "tools/pmt/master/ksfa",
"kmcKsfStandby" : "tools/pmt/standby/ksfb",
"inferMode" : "standard",
"interCommTLSEnabled" : false,
"interCommPort" : 1121,
"interCommTlsCaPath" : "security/grpc/ca/",
"interCommTlsCaFiles" : ["ca.pem"],
"interCommTlsCert" : "security/grpc/certs/server.pem",
"interCommPk" : "security/grpc/keys/server.key.pem",
"interCommPkPwd" : "security/grpc/pass/key_pwd.txt",
"interCommTlsCrlPath" : "security/grpc/certs/",
"interCommTlsCrlFiles" : ["server_crl.pem"],
"openAiSupport" : "vllm",
"tokenTimeout":600,
"e2eTimeout":600
},
"BackendConfig" : {
"backendName" : "mindieservice_llm_engine",
"modelInstanceNumber" : 1,
"npuDeviceIds" : [[0,1,2,3,4,5,6,7]],
"tokenizerProcessNumber" : 8,
"multiNodesInferEnabled" : true,
"multiNodesInferPort" : 1120,
"interNodeTLSEnabled" : false,
"interNodeTlsCaPath" : "security/grpc/ca/",
"interNodeTlsCaFiles" : ["ca.pem"],
"interNodeTlsCert" : "security/grpc/certs/server.pem",
"interNodeTlsPk" : "security/grpc/keys/server.key.pem",
"interNodeTlsPkPwd" : "security/grpc/pass/mindie_server_key_pwd.txt",
"interNodeTlsCrlPath" : "security/grpc/certs/",
"interNodeTlsCrlFiles" : ["server_crl.pem"],
"interNodeKmcKsfMaster" : "tools/pmt/master/ksfa",
"interNodeKmcKsfStandby" : "tools/pmt/standby/ksfb",
"ModelDeployConfig" : {
"maxSeqLen" : 32768,
"maxInputTokenLen" : 32768,
"truncation" : true,
"ModelConfig" : [
{
"modelInstanceType" : "Standard",
"modelName" : "deepseek-r1",
"modelWeightPath" : "/workspace/weights/",
"worldSize" : 8,
"cpuMemSize" : 5,
"npuMemSize" : -1,
"backendType" : "atb",
"trustRemoteCode" : false
}
]
},
"ScheduleConfig" : {
"templateType" : "Standard",
"templateName" : "Standard_LLM",
"cacheBlockSize" : 128,
"maxPrefillBatchSize" : 8,
"maxPrefillTokens" : 32768,
"prefillTimeMsPerReq" : 150,
"prefillPolicyType" : 0,
"decodeTimeMsPerReq" : 50,
"decodePolicyType" : 0,
"maxBatchSize" : 8,
"maxIterTimes" : 1024,
"maxPreemptCount" : 0,
"supportSelectBatch" : false,
"maxQueueDelayMicroseconds" : 5000
}
}
}
两台机器的rank_table_file.json内容如下:
{
"version": "1.0",
"server_count": "2",
"server_list": [
{
"server_id": "10.199.117.23",
"container_ip": "10.199.117.23",
"device": [
{"device_id": "0","device_ip": "10.20.0.10","rank_id": "0"},
{"device_id": "1","device_ip": "10.20.0.11","rank_id": "1"},
{"device_id": "2","device_ip": "10.20.0.12","rank_id": "2"},
{"device_id": "3","device_ip": "10.20.0.13","rank_id": "3"},
{"device_id": "4","device_ip": "10.20.0.14","rank_id": "4"},
{"device_id": "5","device_ip": "10.20.0.15","rank_id": "5"},
{"device_id": "6","device_ip": "10.20.0.16","rank_id": "6"},
{"device_id": "7","device_ip": "10.20.0.17","rank_id": "7"}
]
},
{
"server_id": "10.199.117.25",
"container_ip": "10.199.117.25",
"device": [
{"device_id": "0","device_ip": "10.20.0.20","rank_id": "8"},
{"device_id": "1","device_ip": "10.20.0.21","rank_id": "9"},
{"device_id": "2","device_ip": "10.20.0.22","rank_id": "10"},
{"device_id": "3","device_ip": "10.20.0.23","rank_id": "11"},
{"device_id": "4","device_ip": "10.20.0.24","rank_id": "12"},
{"device_id": "5","device_ip": "10.20.0.25","rank_id": "13"},
{"device_id": "6","device_ip": "10.20.0.26","rank_id": "14"},
{"device_id": "7","device_ip": "10.20.0.27","rank_id": "15"}
]
}
],
"status": "completed"
}
目前论坛有关帖子都没有可行解决方案,需求比较急,希望能够沟通有效解决方案。