从官网拉取的镜像，1.0.RC2-300I-Duo-arm64 8月4号的版本，按照操作流程执行了source install_and_enable_cann.sh，并在~/.bashrc配置了source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/mindie/set_env.sh
source /usr/local/Ascend/llm_model/set_env.sh
启动容器
docker run --name MindService -it --net=host --shm-size=500g \
--privileged=true \
--device=/dev/davinci_manager \
--device=/dev/hisi_hdc \
--device=/dev/devmm_svm \
--entrypoint=bash \
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
-v /usr/local/dcmi:/usr/local/dcmi \
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-v /usr/local/sbin:/usr/local/sbin \
-v /etc/ascend_install.info:/etc/ascend_install.info \
-v /etc/vnpu.cfg:/etc/vnpu.cfg \
-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-v /root/llm:/root/llm \
-v /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime \
-v /tmp:/tmp \
-v /home:/home \
-w /home \
-v /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime \
50f78f73681d
其中 /root/llm下存放的是qwen1.5-7B-Chat模型
mindservice的配置文件如下：
{
"OtherParam" :
{
"ResourceParam" :
{
"cacheBlockSize" : 128
},
"LogParam" :
{
"logLevel" : "Info",
"logPath" : "logs/mindservice.log"
},
"ServeParam" :
{
"ipAddress" : "127.0.0.1",
"managementIpAddress" : "127.0.0.2",
"port" : 1025,
"managementPort" : 1026,
"maxLinkNum" : 300,
"httpsEnabled" : false,
"tlsCaPath" : "security/ca/",
"tlsCaFile" : ["ca.pem"],
"tlsCert" : "security/certs/server.pem",
"tlsPk" : "security/keys/server.key.pem",
"tlsPkPwd" : "security/pass/mindie_server_key_pwd.txt",
"tlsCrl" : "security/certs/server_crl.pem",
"managementTlsCaFile" : ["management_ca.pem"],
"managementTlsCert" : "security/certs/management_server.pem",
"managementTlsPk" : "security/keys/management_server.key.pem",
"managementTlsPkPwd" : "security/pass/management_mindie_server_key_pwd.txt",
"managementTlsCrl" : "security/certs/management_server_crl.pem",
"kmcKsfMaster" : "tools/pmt/master/ksfa",
"kmcKsfStandby" : "tools/pmt/standby/ksfb",
"multiNodesInferPort" : 1120,
"interNodeTLSEnabled" : true,
"interNodeTlsCaFile" : "security/ca/ca.pem",
"interNodeTlsCert" : "security/certs/server.pem",
"interNodeTlsPk" : "security/keys/server.key.pem",
"interNodeTlsPkPwd" : "security/pass/mindie_server_key_pwd.txt",
"interNodeKmcKsfMaster" : "tools/pmt/master/ksfa",
"interNodeKmcKsfStandby" : "tools/pmt/standby/ksfb"
}
},
"WorkFlowParam" :
{
"TemplateParam" :
{
"templateType" : "Standard",
"templateName" : "Standard_llama"
}
},
"ModelDeployParam" :
{
"engineName" : "mindieservice_llm_engine",
"modelInstanceNumber" : 1,
"tokenizerProcessNumber" : 8,
"maxSeqLen" : 2560,
"npuDeviceIds" : [[0,1]],
"multiNodesInferEnabled" : false,
"ModelParam" : [
{
"modelInstanceType" : "Standard",
"modelName" : "qwen1.5-7B-Chat",
"modelWeightPath" : "/root/llm/qwen1.5-7B-Chat",
"worldSize" : 2,
"cpuMemSize" : 5,
"npuMemSize" : 8,
"backendType" : "atb",
"pluginParams" : ""
}
]
},
"ScheduleParam" :
{
"maxPrefillBatchSize" : 50,
"maxPrefillTokens" : 8192,
"prefillTimeMsPerReq" : 150,
"prefillPolicyType" : 0,
"decodeTimeMsPerReq" : 50,
"decodePolicyType" : 0,
"maxBatchSize" : 200,
"maxIterTimes" : 512,
"maxPreemptCount" : 0,
"supportSelectBatch" : false,
"maxQueueDelayMicroseconds" : 5000
}
}
然后执行 ./bin/mindieservice_daemon
报错内容如下：
raceback (most recent call last):
File "/usr/local/python3.10.2/lib/python3.10/site-packages/model_wrapper/model.py", line 28, in initialize
return self.python_model.initialize(config)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/model_wrapper/standard_model.py", line 28, in initialize
self.generator = Generator(
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/generator.py", line 56, in __init__
self.warm_up(max_prefill_tokens, max_seq_len, max_iter_times)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/generator.py", line 133, in warm_up
self.generator_backend.warm_up(model_inputs)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/adapter/generator_backend.py", line 42, in warm_up
_ = self.forward(model_inputs, **kwargs)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 52, in forward
logits = self.model_wrapper.forward(model_inputs, self.cache_pool.npu_cache, **kwargs)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 60, in forward
logits = self.forward_tensor(
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 86, in forward_tensor
logits = self.model_runner.forward(
File "/usr/local/Ascend/llm_model/atb_llm/runner/model_runner.py", line 100, in forward
return self.model.forward(**kwargs)
File "/usr/local/Ascend/llm_model/atb_llm/models/base/flash_causal_lm.py", line 229, in forward
logits = self.execute_ascend_operator(acl_inputs, acl_param, is_prefill)
File "/usr/local/Ascend/llm_model/atb_llm/models/base/flash_causal_lm.py", line 200, in execute_ascend_operator
acl_model_out = self.acl_encoder_operation.execute(acl_inputs, acl_param)
RuntimeError: execute\u5931\u8d25 \u8be6\u7ec6\u4fe1\u606f\u89c1Ascend\u5b98\u65b9\u6587\u6863\uff0c\u8bf7\u5f00\u542f\u65e5\u5fd7\u8fdb\u4e00\u6b65\u5b9a\u4f4d\u95ee\u9898\u3002
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/python3.10.2/lib/python3.10/logging/__init__.py", line 1103, in emit
stream.write(msg + self.terminator)
UnicodeEncodeError: 'ascii' codec can't encode characters in position 75-76: ordinal not in range(128)
Call stack:
File "/usr/local/python3.10.2/lib/python3.10/site-packages/model_wrapper/model.py", line 30, in initialize
logger.exception(f"[Model]\t>>> Exception:{e}")
Message: '[Model]\t>>> Exception:execute\u5931\u8d25 \u8be6\u7ec6\u4fe1\u606f\u89c1Ascend\u5b98\u65b9\u6587\u6863\uff0c\u8bf7\u5f00\u542f\u65e5\u5fd7\u8fdb\u4e00\u6b65\u5b9a\u4f4d\u95ee\u9898\u3002\n'
Arguments: ()
Traceback (most recent call last):
File "/usr/local/python3.10.2/lib/python3.10/site-packages/model_wrapper/model.py", line 28, in initialize
return self.python_model.initialize(config)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/model_wrapper/standard_model.py", line 28, in initialize
self.generator = Generator(
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/generator.py", line 56, in __init__
self.warm_up(max_prefill_tokens, max_seq_len, max_iter_times)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/generator.py", line 133, in warm_up
self.generator_backend.warm_up(model_inputs)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/adapter/generator_backend.py", line 42, in warm_up
_ = self.forward(model_inputs, **kwargs)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 52, in forward
logits = self.model_wrapper.forward(model_inputs, self.cache_pool.npu_cache, **kwargs)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 60, in forward
logits = self.forward_tensor(
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 86, in forward_tensor
logits = self.model_runner.forward(
File "/usr/local/Ascend/llm_model/atb_llm/runner/model_runner.py", line 100, in forward
return self.model.forward(**kwargs)
File "/usr/local/Ascend/llm_model/atb_llm/models/base/flash_causal_lm.py", line 229, in forward
logits = self.execute_ascend_operator(acl_inputs, acl_param, is_prefill)
File "/usr/local/Ascend/llm_model/atb_llm/models/base/flash_causal_lm.py", line 200, in execute_ascend_operator
acl_model_out = self.acl_encoder_operation.execute(acl_inputs, acl_param)
RuntimeError: execute\u5931\u8d25 \u8be6\u7ec6\u4fe1\u606f\u89c1Ascend\u5b98\u65b9\u6587\u6863\uff0c\u8bf7\u5f00\u542f\u65e5\u5fd7\u8fdb\u4e00\u6b65\u5b9a\u4f4d\u95ee\u9898\u3002
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/python3.10.2/lib/python3.10/logging/__init__.py", line 1103, in emit
stream.write(msg + self.terminator)
UnicodeEncodeError: 'ascii' codec can't encode characters in position 75-76: ordinal not in range(128)
Call stack:
File "/usr/local/python3.10.2/lib/python3.10/site-packages/model_wrapper/model.py", line 30, in initialize
logger.exception(f"[Model]\t>>> Exception:{e}")
Message: '[Model]\t>>> Exception:execute\u5931\u8d25 \u8be6\u7ec6\u4fe1\u606f\u89c1Ascend\u5b98\u65b9\u6587\u6863\uff0c\u8bf7\u5f00\u542f\u65e5\u5fd7\u8fdb\u4e00\u6b65\u5b9a\u4f4d\u95ee\u9898\u3002\n'
Arguments: ()
Fatal Python error: PyThreadState_Get: the function must be called with the GIL held, but the GIL is released (the current Python thread state is NULL)
Python runtime state: finalizing (tstate=0x0000fffd7401d570)
Fatal Python error: PyThreadState_Get: the function must be called with the GIL held, but the GIL is released (the current Python thread state is NULL)
Python runtime state: finalizing (tstate=0x0000fffef801d570)
Daemon wait pid with 18329, status 134Daemon is killing...
Killed
点赞
收藏
回复
分享
举报
已解决
从官网拉取的镜像，1.0.RC2-300I-Duo-arm64 8月4号的版本，按照操作流程执行了source install_and_enable_cann.sh，并在~/.bashrc配置了source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/mindie/set_env.sh
source /usr/local/Ascend/llm_model/set_env.sh
启动容器
docker run --name MindService -it --net=host --shm-size=500g \
--privileged=true \
--device=/dev/davinci_manager \
--device=/dev/hisi_hdc \
--device=/dev/devmm_svm \
--entrypoint=bash \
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
-v /usr/local/dcmi:/usr/local/dcmi \
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-v /usr/local/sbin:/usr/local/sbin \
-v /etc/ascend_install.info:/etc/ascend_install.info \
-v /etc/vnpu.cfg:/etc/vnpu.cfg \
-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-v /root/llm:/root/llm \
-v /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime \
-v /tmp:/tmp \
-v /home:/home \
-w /home \
-v /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime \
50f78f73681d
其中 /root/llm下存放的是qwen1.5-7B-Chat模型
mindservice的配置文件如下：
{
"OtherParam" :
{
"ResourceParam" :
{
"cacheBlockSize" : 128
},
"LogParam" :
{
"logLevel" : "Info",
"logPath" : "logs/mindservice.log"
},
"ServeParam" :
{
"ipAddress" : "127.0.0.1",
"managementIpAddress" : "127.0.0.2",
"port" : 1025,
"managementPort" : 1026,
"maxLinkNum" : 300,
"httpsEnabled" : false,
"tlsCaPath" : "security/ca/",
"tlsCaFile" : ["ca.pem"],
"tlsCert" : "security/certs/server.pem",
"tlsPk" : "security/keys/server.key.pem",
"tlsPkPwd" : "security/pass/mindie_server_key_pwd.txt",
"tlsCrl" : "security/certs/server_crl.pem",
"managementTlsCaFile" : ["management_ca.pem"],
"managementTlsCert" : "security/certs/management_server.pem",
"managementTlsPk" : "security/keys/management_server.key.pem",
"managementTlsPkPwd" : "security/pass/management_mindie_server_key_pwd.txt",
"managementTlsCrl" : "security/certs/management_server_crl.pem",
"kmcKsfMaster" : "tools/pmt/master/ksfa",
"kmcKsfStandby" : "tools/pmt/standby/ksfb",
"multiNodesInferPort" : 1120,
"interNodeTLSEnabled" : true,
"interNodeTlsCaFile" : "security/ca/ca.pem",
"interNodeTlsCert" : "security/certs/server.pem",
"interNodeTlsPk" : "security/keys/server.key.pem",
"interNodeTlsPkPwd" : "security/pass/mindie_server_key_pwd.txt",
"interNodeKmcKsfMaster" : "tools/pmt/master/ksfa",
"interNodeKmcKsfStandby" : "tools/pmt/standby/ksfb"
}
},
"WorkFlowParam" :
{
"TemplateParam" :
{
"templateType" : "Standard",
"templateName" : "Standard_llama"
}
},
"ModelDeployParam" :
{
"engineName" : "mindieservice_llm_engine",
"modelInstanceNumber" : 1,
"tokenizerProcessNumber" : 8,
"maxSeqLen" : 2560,
"npuDeviceIds" : [[0,1]],
"multiNodesInferEnabled" : false,
"ModelParam" : [
{
"modelInstanceType" : "Standard",
"modelName" : "qwen1.5-7B-Chat",
"modelWeightPath" : "/root/llm/qwen1.5-7B-Chat",
"worldSize" : 2,
"cpuMemSize" : 5,
"npuMemSize" : 8,
"backendType" : "atb",
"pluginParams" : ""
}
]
},
"ScheduleParam" :
{
"maxPrefillBatchSize" : 50,
"maxPrefillTokens" : 8192,
"prefillTimeMsPerReq" : 150,
"prefillPolicyType" : 0,
"decodeTimeMsPerReq" : 50,
"decodePolicyType" : 0,
"maxBatchSize" : 200,
"maxIterTimes" : 512,
"maxPreemptCount" : 0,
"supportSelectBatch" : false,
"maxQueueDelayMicroseconds" : 5000
}
}
然后执行 ./bin/mindieservice_daemon
报错内容如下：
raceback (most recent call last):
File "/usr/local/python3.10.2/lib/python3.10/site-packages/model_wrapper/model.py", line 28, in initialize
return self.python_model.initialize(config)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/model_wrapper/standard_model.py", line 28, in initialize
self.generator = Generator(
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/generator.py", line 56, in __init__
self.warm_up(max_prefill_tokens, max_seq_len, max_iter_times)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/generator.py", line 133, in warm_up
self.generator_backend.warm_up(model_inputs)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/adapter/generator_backend.py", line 42, in warm_up
_ = self.forward(model_inputs, **kwargs)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 52, in forward
logits = self.model_wrapper.forward(model_inputs, self.cache_pool.npu_cache, **kwargs)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 60, in forward
logits = self.forward_tensor(
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 86, in forward_tensor
logits = self.model_runner.forward(
File "/usr/local/Ascend/llm_model/atb_llm/runner/model_runner.py", line 100, in forward
return self.model.forward(**kwargs)
File "/usr/local/Ascend/llm_model/atb_llm/models/base/flash_causal_lm.py", line 229, in forward
logits = self.execute_ascend_operator(acl_inputs, acl_param, is_prefill)
File "/usr/local/Ascend/llm_model/atb_llm/models/base/flash_causal_lm.py", line 200, in execute_ascend_operator
acl_model_out = self.acl_encoder_operation.execute(acl_inputs, acl_param)
RuntimeError: execute\u5931\u8d25 \u8be6\u7ec6\u4fe1\u606f\u89c1Ascend\u5b98\u65b9\u6587\u6863\uff0c\u8bf7\u5f00\u542f\u65e5\u5fd7\u8fdb\u4e00\u6b65\u5b9a\u4f4d\u95ee\u9898\u3002
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/python3.10.2/lib/python3.10/logging/__init__.py", line 1103, in emit
stream.write(msg + self.terminator)
UnicodeEncodeError: 'ascii' codec can't encode characters in position 75-76: ordinal not in range(128)
Call stack:
File "/usr/local/python3.10.2/lib/python3.10/site-packages/model_wrapper/model.py", line 30, in initialize
logger.exception(f"[Model]\t>>> Exception:{e}")
Message: '[Model]\t>>> Exception:execute\u5931\u8d25 \u8be6\u7ec6\u4fe1\u606f\u89c1Ascend\u5b98\u65b9\u6587\u6863\uff0c\u8bf7\u5f00\u542f\u65e5\u5fd7\u8fdb\u4e00\u6b65\u5b9a\u4f4d\u95ee\u9898\u3002\n'
Arguments: ()
Traceback (most recent call last):
File "/usr/local/python3.10.2/lib/python3.10/site-packages/model_wrapper/model.py", line 28, in initialize
return self.python_model.initialize(config)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/model_wrapper/standard_model.py", line 28, in initialize
self.generator = Generator(
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/generator.py", line 56, in __init__
self.warm_up(max_prefill_tokens, max_seq_len, max_iter_times)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/generator.py", line 133, in warm_up
self.generator_backend.warm_up(model_inputs)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/adapter/generator_backend.py", line 42, in warm_up
_ = self.forward(model_inputs, **kwargs)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 52, in forward
logits = self.model_wrapper.forward(model_inputs, self.cache_pool.npu_cache, **kwargs)
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 60, in forward
logits = self.forward_tensor(
File "/usr/local/python3.10.2/lib/python3.10/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 86, in forward_tensor
logits = self.model_runner.forward(
File "/usr/local/Ascend/llm_model/atb_llm/runner/model_runner.py", line 100, in forward
return self.model.forward(**kwargs)
File "/usr/local/Ascend/llm_model/atb_llm/models/base/flash_causal_lm.py", line 229, in forward
logits = self.execute_ascend_operator(acl_inputs, acl_param, is_prefill)
File "/usr/local/Ascend/llm_model/atb_llm/models/base/flash_causal_lm.py", line 200, in execute_ascend_operator
acl_model_out = self.acl_encoder_operation.execute(acl_inputs, acl_param)
RuntimeError: execute\u5931\u8d25 \u8be6\u7ec6\u4fe1\u606f\u89c1Ascend\u5b98\u65b9\u6587\u6863\uff0c\u8bf7\u5f00\u542f\u65e5\u5fd7\u8fdb\u4e00\u6b65\u5b9a\u4f4d\u95ee\u9898\u3002
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/python3.10.2/lib/python3.10/logging/__init__.py", line 1103, in emit
stream.write(msg + self.terminator)
UnicodeEncodeError: 'ascii' codec can't encode characters in position 75-76: ordinal not in range(128)
Call stack:
File "/usr/local/python3.10.2/lib/python3.10/site-packages/model_wrapper/model.py", line 30, in initialize
logger.exception(f"[Model]\t>>> Exception:{e}")
Message: '[Model]\t>>> Exception:execute\u5931\u8d25 \u8be6\u7ec6\u4fe1\u606f\u89c1Ascend\u5b98\u65b9\u6587\u6863\uff0c\u8bf7\u5f00\u542f\u65e5\u5fd7\u8fdb\u4e00\u6b65\u5b9a\u4f4d\u95ee\u9898\u3002\n'
Arguments: ()
Fatal Python error: PyThreadState_Get: the function must be called with the GIL held, but the GIL is released (the current Python thread state is NULL)
Python runtime state: finalizing (tstate=0x0000fffd7401d570)
Fatal Python error: PyThreadState_Get: the function must be called with the GIL held, but the GIL is released (the current Python thread state is NULL)
Python runtime state: finalizing (tstate=0x0000fffef801d570)
Daemon wait pid with 18329, status 134Daemon is killing...
Killed