环境：Ubuntu20.04。内核5.0.4-26.python=3.10. torch=2.1.0+cpu， torch_npu=2.1.0.post3依赖包完整，mindie推理，无异常。双卡推理报错如下。

报错信息如下：

2024-07-25 15:47:46,031 [INFO] [pid: 243695] logging.py-55: trans to 29
2024-07-25 15:47:46,031 [INFO] [pid: 243695] logging.py-55: trans to 29
2024-07-25 15:47:46,031 [INFO] [pid: 243695] logging.py-55: trans to 2
2024-07-25 15:47:46,035 [INFO] [pid: 243695] logging.py-55: trans to 29
2024-07-25 15:47:46,060 [INFO] [pid: 243695] logging.py-55: <<<<<<< ori k_caches[0].shape=torch.Size([12, 64, 128, 16])
2024-07-25 15:47:46,157 [INFO] [pid: 243695] flash_causal_lm.py-143: <<<<<<<after transdata k_caches[0].shape=torch.Size([12, 64, 128, 16])
2024-07-25 15:47:46,158 [INFO] [pid: 243695] logging.py-55: >>>>>>id of kcache is 140003481682784 id of vcache is 140000965161936
2024-07-25 15:47:46,164 [INFO] [pid: 243697] flash_causal_lm.py-143: <<<<<<<after transdata k_caches[0].shape=torch.Size([12, 64, 128, 16])
Traceback (most recent call last):
  File "/root/miniconda3/envs/qmindie/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/root/miniconda3/envs/qmindie/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/usr/local/Ascend/atb_models/examples/run_pa.py", line 330, in <module>
    pa_runner.warm_up()
  File "/usr/local/Ascend/atb_models/examples/run_pa.py", line 140, in warm_up
    logits = self.model.forward(
  File "/usr/local/Ascend/atb_models/atb_llm/runner/model_runner.py", line 89, in forward
    return self.model.forward(**kwargs)
  File "/usr/local/Ascend/atb_models/atb_llm/models/base/flash_causal_lm.py", line 240, in forward
    logits = self.execute_ascend_operator(acl_inputs, acl_param, is_prefill)
  File "/usr/local/Ascend/atb_models/atb_llm/models/base/flash_causal_lm.py", line 218, in execute_ascend_operator
    acl_hidden_state = acl_model_out[0]
IndexError: list index out of range
Traceback (most recent call last):
  File "/root/miniconda3/envs/qmindie/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/root/miniconda3/envs/qmindie/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/usr/local/Ascend/atb_models/examples/run_pa.py", line 330, in <module>
    pa_runner.warm_up()
  File "/usr/local/Ascend/atb_models/examples/run_pa.py", line 140, in warm_up
    logits = self.model.forward(
  File "/usr/local/Ascend/atb_models/atb_llm/runner/model_runner.py", line 89, in forward
    return self.model.forward(**kwargs)
  File "/usr/local/Ascend/atb_models/atb_llm/models/base/flash_causal_lm.py", line 240, in forward
    logits = self.execute_ascend_operator(acl_inputs, acl_param, is_prefill)
  File "/usr/local/Ascend/atb_models/atb_llm/models/base/flash_causal_lm.py", line 218, in execute_ascend_operator
    acl_hidden_state = acl_model_out[0]
IndexError: list index out of range

config配置如下。

 

{ 

    "OtherParam": 

    { 

        "ResourceParam" : 

        { 

            "cacheBlockSize" : 128, 

            "preAllocBlocks" : 4 

        }, 

        "LogParam" : 

        { 

            "logLevel" : "Info", 

            "logPath" : "/logs/mindservice.log" 

        }, 

        "ServeParam" : 

        { 

            "ipAddress" : "127.0.0.1", 

            "port" : 1025, 

            "maxLinkNum" : 300, 

            "httpsEnabled" : false, 

            "tlsCaPath" : "security/ca/", 

            "tlsCaFile" : ["ca.pem"], 

            "tlsCert" : "security/certs/server.pem", 

            "tlsPk" : "security/keys/server.key.pem", 

            "tlsPkPwd" : "security/pass/mindie_server_key_pwd.txt", 

            "kmcKsfMaster" : "tools/pmt/master/ksfa", 

            "kmcKsfStandby" : "tools/pmt/standby/ksfb", 

            "tlsCrl" : "security/certs/server_crl.pem" 

        } 

    }, 

    "WorkFlowParam": 

    { 

        "TemplateParam" : 

        { 

            "templateType": "Standard", 

            "templateName" : "Standard_llama", 

            "pipelineNumber" : 1 

        } 

    }, 

    "ModelDeployParam": 

    { 

        "maxSeqLen" : 2560, 

        "npuDeviceIds" : [[0,1,2,3]], 

        "ModelParam" : [ 

            { 

                "modelInstanceType": "Standard", 

                "modelName" : "llama-2-7b", 

                "modelWeightPath" : "/data/llama-2-7b-hf", 

                "worldSize" : 4, 

                "cpuMemSize" : 5, 

                "npuMemSize" : 8, 

                "backendType": "atb" 

            } 

        ] 

    }, 

    "ScheduleParam": 

    { 

        "maxPrefillBatchSize" : 50, 

        "maxPrefillTokens" : 8192, 

        "prefillTimeMsPerReq" : 150, 

        "prefillPolicyType" : 0, 

 

        "decodeTimeMsPerReq" : 50, 

        "decodePolicyType" : 0, 

 

        "maxBatchSize" : 200, 

        "maxIterTimes" : 512, 

        "maxPreemptCount" : 200, 

        "supportSelectBatch" : false, 

        "maxQueueDelayMicroseconds" : 5000 

    } 

}