华为计算微信公众号
昇腾AI开发者公众号
华为计算微博
华为计算今日头条
- 使用镜像 swr.cn-south-1.myhuaweicloud.com/ascendhub/mindie:2.1.RC2-800I-A2-py311-openeuler24.03-lts
- 部署推理 qwen2.5-vl-7b 模型，启动和使用正常
问题：部署完成后，刚开始使用的前几天速度正常，在使用一段时间后，不清楚为什么推理速度突然莫名奇妙的变慢，推理进程没动过，显存占用也未见异常，测试时使用的是完全相同的图片和prompt，速度正常时耗时 3~5s，速度慢时耗时 15~20s
在 64G 显存的 910B3 和 32G 显存的 910B4 两台不同服务器上都遇到该问题
下面是使用的配置 ./conf/config.json 完整内容：
{ "Version": "1.0.0", "ServerConfig": { "ipAddress": "0.0.0.0", "managementIpAddress": "127.0.0.99", "port": 19300, "managementPort": 14710, "metricsPort": 14720, "allowAllZeroIpListening": true, "maxLinkNum": 1000, "httpsEnabled": false, "fullTextEnabled": false, "tlsCaPath": "security/ca/", "tlsCaFile": [ "ca.pem" ], "tlsCert": "security/certs/server.pem", "tlsPk": "security/keys/server.key.pem", "tlsPkPwd": "security/pass/key_pwd.txt", "tlsCrlPath": "security/certs/", "tlsCrlFiles": [ "server_crl.pem" ], "managementTlsCaFile": [ "management_ca.pem" ], "managementTlsCert": "security/certs/management/server.pem", "managementTlsPk": "security/keys/management/server.key.pem", "managementTlsPkPwd": "security/pass/management/key_pwd.txt", "managementTlsCrlPath": "security/management/certs/", "managementTlsCrlFiles": [ "server_crl.pem" ], "kmcKsfMaster": "tools/pmt/master/ksfa", "kmcKsfStandby": "tools/pmt/standby/ksfb", "inferMode": "standard", "interCommTLSEnabled": true, "interCommPort": 1121, "interCommTlsCaPath": "security/grpc/ca/", "interCommTlsCaFiles": [ "ca.pem" ], "interCommTlsCert": "security/grpc/certs/server.pem", "interCommPk": "security/grpc/keys/server.key.pem", "interCommPkPwd": "security/grpc/pass/key_pwd.txt", "interCommTlsCrlPath": "security/grpc/certs/", "interCommTlsCrlFiles": [ "server_crl.pem" ], "openAiSupport": "vllm", "tokenTimeout": 600, "e2eTimeout": 600, "distDPServerEnabled": false }, "BackendConfig": { "backendName": "mindieservice_llm_engine", "modelInstanceNumber": 1, "npuDeviceIds": [ [ 0, 1 ] ], "tokenizerProcessNumber": 8, "multiNodesInferEnabled": false, "multiNodesInferPort": 1120, "interNodeTLSEnabled": true, "interNodeTlsCaPath": "security/grpc/ca/", "interNodeTlsCaFiles": [ "ca.pem" ], "interNodeTlsCert": "security/grpc/certs/server.pem", "interNodeTlsPk": "security/grpc/keys/server.key.pem", "interNodeTlsPkPwd": "security/grpc/pass/mindie_server_key_pwd.txt", "interNodeTlsCrlPath": "security/grpc/certs/", "interNodeTlsCrlFiles": [ "server_crl.pem" ], "interNodeKmcKsfMaster": "tools/pmt/master/ksfa", "interNodeKmcKsfStandby": "tools/pmt/standby/ksfb", "ModelDeployConfig": { "maxSeqLen": 20000, "maxInputTokenLen": 15000, "truncation": false, "ModelConfig": [ { "modelInstanceType": "Standard", "modelName": "qwen2.5-vl-7b", "modelWeightPath": "/root/data/models/Qwen2.5-VL-7B-Instruct", "worldSize": 2, "cpuMemSize": 5, "npuMemSize": 16, "backendType": "atb", "trustRemoteCode": false, "async_scheduler_wait_time": 120, "kv_trans_timeout": 10, "kv_link_timeout": 1080 } ] }, "ScheduleConfig": { "templateType": "Standard", "templateName": "Standard_LLM", "cacheBlockSize": 128, "maxPrefillBatchSize": 50, "maxPrefillTokens": 20000, "prefillTimeMsPerReq": 150, "prefillPolicyType": 0, "decodeTimeMsPerReq": 50, "decodePolicyType": 0, "maxBatchSize": 100, "maxIterTimes": 4096, "maxPreemptCount": 0, "supportSelectBatch": false, "maxQueueDelayMicroseconds": 5000 } } }
下面是镜像启动 docker-compose.yaml 文件：
services: mindie-test: container_name: mindie-test image: swr.cn-south-1.myhuaweicloud.com/ascendhub/mindie:2.1.RC2-800I-A2-py311-openeuler24.03-lts privileged: true shm_size: '1gb' network_mode: host ipc: host stdin_open: true tty: true volumes: # 驱动等服务 - /usr/local/Ascend/driver:/usr/local/Ascend/driver - /usr/local/sbin:/usr/local/sbin # 模型权重等 - /root/data/models:/root/data/models command: /bin/bash
我要发帖子
点赞
收藏
回复
分享
举报
已解决 已解决
- 使用镜像 swr.cn-south-1.myhuaweicloud.com/ascendhub/mindie:2.1.RC2-800I-A2-py311-openeuler24.03-lts
- 部署推理 qwen2.5-vl-7b 模型，启动和使用正常
问题：部署完成后，刚开始使用的前几天速度正常，在使用一段时间后，不清楚为什么推理速度突然莫名奇妙的变慢，推理进程没动过，显存占用也未见异常，测试时使用的是完全相同的图片和prompt，速度正常时耗时 3~5s，速度慢时耗时 15~20s
在 64G 显存的 910B3 和 32G 显存的 910B4 两台不同服务器上都遇到该问题
下面是使用的配置 ./conf/config.json 完整内容：
{ "Version": "1.0.0", "ServerConfig": { "ipAddress": "0.0.0.0", "managementIpAddress": "127.0.0.99", "port": 19300, "managementPort": 14710, "metricsPort": 14720, "allowAllZeroIpListening": true, "maxLinkNum": 1000, "httpsEnabled": false, "fullTextEnabled": false, "tlsCaPath": "security/ca/", "tlsCaFile": [ "ca.pem" ], "tlsCert": "security/certs/server.pem", "tlsPk": "security/keys/server.key.pem", "tlsPkPwd": "security/pass/key_pwd.txt", "tlsCrlPath": "security/certs/", "tlsCrlFiles": [ "server_crl.pem" ], "managementTlsCaFile": [ "management_ca.pem" ], "managementTlsCert": "security/certs/management/server.pem", "managementTlsPk": "security/keys/management/server.key.pem", "managementTlsPkPwd": "security/pass/management/key_pwd.txt", "managementTlsCrlPath": "security/management/certs/", "managementTlsCrlFiles": [ "server_crl.pem" ], "kmcKsfMaster": "tools/pmt/master/ksfa", "kmcKsfStandby": "tools/pmt/standby/ksfb", "inferMode": "standard", "interCommTLSEnabled": true, "interCommPort": 1121, "interCommTlsCaPath": "security/grpc/ca/", "interCommTlsCaFiles": [ "ca.pem" ], "interCommTlsCert": "security/grpc/certs/server.pem", "interCommPk": "security/grpc/keys/server.key.pem", "interCommPkPwd": "security/grpc/pass/key_pwd.txt", "interCommTlsCrlPath": "security/grpc/certs/", "interCommTlsCrlFiles": [ "server_crl.pem" ], "openAiSupport": "vllm", "tokenTimeout": 600, "e2eTimeout": 600, "distDPServerEnabled": false }, "BackendConfig": { "backendName": "mindieservice_llm_engine", "modelInstanceNumber": 1, "npuDeviceIds": [ [ 0, 1 ] ], "tokenizerProcessNumber": 8, "multiNodesInferEnabled": false, "multiNodesInferPort": 1120, "interNodeTLSEnabled": true, "interNodeTlsCaPath": "security/grpc/ca/", "interNodeTlsCaFiles": [ "ca.pem" ], "interNodeTlsCert": "security/grpc/certs/server.pem", "interNodeTlsPk": "security/grpc/keys/server.key.pem", "interNodeTlsPkPwd": "security/grpc/pass/mindie_server_key_pwd.txt", "interNodeTlsCrlPath": "security/grpc/certs/", "interNodeTlsCrlFiles": [ "server_crl.pem" ], "interNodeKmcKsfMaster": "tools/pmt/master/ksfa", "interNodeKmcKsfStandby": "tools/pmt/standby/ksfb", "ModelDeployConfig": { "maxSeqLen": 20000, "maxInputTokenLen": 15000, "truncation": false, "ModelConfig": [ { "modelInstanceType": "Standard", "modelName": "qwen2.5-vl-7b", "modelWeightPath": "/root/data/models/Qwen2.5-VL-7B-Instruct", "worldSize": 2, "cpuMemSize": 5, "npuMemSize": 16, "backendType": "atb", "trustRemoteCode": false, "async_scheduler_wait_time": 120, "kv_trans_timeout": 10, "kv_link_timeout": 1080 } ] }, "ScheduleConfig": { "templateType": "Standard", "templateName": "Standard_LLM", "cacheBlockSize": 128, "maxPrefillBatchSize": 50, "maxPrefillTokens": 20000, "prefillTimeMsPerReq": 150, "prefillPolicyType": 0, "decodeTimeMsPerReq": 50, "decodePolicyType": 0, "maxBatchSize": 100, "maxIterTimes": 4096, "maxPreemptCount": 0, "supportSelectBatch": false, "maxQueueDelayMicroseconds": 5000 } } }下面是镜像启动 docker-compose.yaml 文件：
services: mindie-test: container_name: mindie-test image: swr.cn-south-1.myhuaweicloud.com/ascendhub/mindie:2.1.RC2-800I-A2-py311-openeuler24.03-lts privileged: true shm_size: '1gb' network_mode: host ipc: host stdin_open: true tty: true volumes: # 驱动等服务 - /usr/local/Ascend/driver:/usr/local/Ascend/driver - /usr/local/sbin:/usr/local/sbin # 模型权重等 - /root/data/models:/root/data/models command: /bin/bash