/usr/local/Ascend/mindie/latest/mindie-service# cat conf/config.json
{
    "Version" : "1.1.0",
    "LogConfig" :
    {
        "logLevel" : "Info",
        "logFileSize" : 20,
        "logFileNum" : 20,
        "logPath" : "logs/mindservice.log"
    },

    "ServerConfig" :
    {
        "ipAddress" : "127.0.0.1",
        "managementIpAddress" : "127.0.0.2",
        "port" : 1025,
        "managementPort" : 1026,
        "metricsPort" : 1027,
        "allowAllZeroIpListening" : false,
        "maxLinkNum" : 1000,
        "httpsEnabled" : false,
        "fullTextEnabled" : false,
        "tlsCaPath" : "security/ca/",
        "tlsCaFile" : ["ca.pem"],
        "tlsCert" : "security/certs/server.pem",
        "tlsPk" : "security/keys/server.key.pem",
        "tlsPkPwd" : "security/pass/key_pwd.txt",
        "tlsCrlPath" : "security/certs/",
        "tlsCrlFiles" : ["server_crl.pem"],
        "managementTlsCaFile" : ["management_ca.pem"],
        "managementTlsCert" : "security/certs/management/server.pem",
        "managementTlsPk" : "security/keys/management/server.key.pem",
        "managementTlsPkPwd" : "security/pass/management/key_pwd.txt",
        "managementTlsCrlPath" : "security/management/certs/",
        "managementTlsCrlFiles" : ["server_crl.pem"],
        "kmcKsfMaster" : "tools/pmt/master/ksfa",
        "kmcKsfStandby" : "tools/pmt/standby/ksfb",
        "inferMode" : "standard",
        "interCommTLSEnabled" : true,
        "interCommPort" : 1121,
        "interCommTlsCaPath" : "security/grpc/ca/",
        "interCommTlsCaFiles" : ["ca.pem"],
        "interCommTlsCert" : "security/grpc/certs/server.pem",
        "interCommPk" : "security/grpc/keys/server.key.pem",
        "interCommPkPwd" : "security/grpc/pass/key_pwd.txt",
        "interCommTlsCrlPath" : "security/grpc/certs/",
        "interCommTlsCrlFiles" : ["server_crl.pem"],
        "openAiSupport" : "vllm"
    },

    "BackendConfig" : {
        "backendName" : "mindieservice_llm_engine",
        "modelInstanceNumber" : 1,
        "npuDeviceIds" : [[0,1]],
        "tokenizerProcessNumber" : 8,
        "multiNodesInferEnabled" : false,
        "multiNodesInferPort" : 1120,
        "interNodeTLSEnabled" : true,
        "interNodeTlsCaPath" : "security/grpc/ca/",
        "interNodeTlsCaFiles" : ["ca.pem"],
        "interNodeTlsCert" : "security/grpc/certs/server.pem",
        "interNodeTlsPk" : "security/grpc/keys/server.key.pem",
        "interNodeTlsPkPwd" : "security/grpc/pass/mindie_server_key_pwd.txt",
        "interNodeTlsCrlPath" : "security/grpc/certs/",
        "interNodeTlsCrlFiles" : ["server_crl.pem"],
        "interNodeKmcKsfMaster" : "tools/pmt/master/ksfa",
        "interNodeKmcKsfStandby" : "tools/pmt/standby/ksfb",
        "ModelDeployConfig" :
        {
            "maxSeqLen" : 2560,
            "maxInputTokenLen" : 2048,
            "truncation" : false,
            "ModelConfig" : [
                {
                    "modelInstanceType" : "Standard",
                    "modelName" : "dsr1-distill-qwen7fp16b",
                    "modelWeightPath" : "/data/DeepSeek-R1-Distill-Qwen-7B-fp16/",
                    "worldSize" : 2,
                    "cpuMemSize" : 5,
                    "npuMemSize" : -1,
                    "backendType" : "atb",
                    "trustRemoteCode" : false
                }
            ]
        },

        "ScheduleConfig" :
        {
            "templateType" : "Standard",
            "templateName" : "Standard_LLM",
            "cacheBlockSize" : 128,

            "maxPrefillBatchSize" : 50,
            "maxPrefillTokens" : 8192,
            "prefillTimeMsPerReq" : 150,
            "prefillPolicyType" : 0,

            "decodeTimeMsPerReq" : 50,
            "decodePolicyType" : 0,

            "maxBatchSize" : 200,
            "maxIterTimes" : 512,
            "maxPreemptCount" : 0,
            "supportSelectBatch" : false,
            "maxQueueDelayMicroseconds" : 5000
        }
    }
}

./bin/mindieservice_daemon
2025-02-26 15:06:59,126 [INFO] config.py:30 - model_config {'backend_bin_path': '/usr/local/Ascend/mindie/1.0.0/mindie-llm/bin/', 'backend_log_file': '/usr/local/Ascend/mindie/1.0.0/mindie-service/logs/mindservice.log', 'backend_modelInstance_id': '0', 'backend_type': 'atb', 'block_size': '128', 'cpu_mem': '5', 'deploy_type': 'INTER_PROCESS', 'dp': '0', 'ep': '0', 'executor_type': 'LLM_EXECUTOR_PYTHON', 'globalRankIds': '', 'globalWorldSize': '0', 'interNodeKmcKsfMaster': 'tools/pmt/master/ksfa', 'interNodeKmcKsfStandby': 'tools/pmt/standby/ksfb', 'interNodeTLSEnabled': '1', 'interNodeTlsCaFiles': 'ca.pem,', 'interNodeTlsCaPath': 'security/grpc/ca/', 'interNodeTlsCert': 'security/grpc/certs/server.pem', 'interNodeTlsCrlFiles': 'server_crl.pem,', 'interNodeTlsCrlPath': 'security/grpc/certs/', 'interNodeTlsPk': 'security/grpc/keys/server.key.pem', 'interNodeTlsPkPwd': 'security/grpc/pass/mindie_server_key_pwd.txt', 'isMaster': '0', 'localIP': '', 'local_rank': '1', 'log_error': '1', 'log_file_num': '20', 'log_file_size': '20', 'log_info': '1', 'log_verbose': '0', 'log_warning': '1', 'masterIP': '', 'max_input_len': '2048', 'max_iter_times': '512', 'max_prefill_tokens': '8192', 'max_seq_len': '2560', 'model_cut_policy': 'standard', 'model_id': '/data/ollama-data/hf-mirror/Qwen2.5-32B-Instruct-W8A8/', 'model_instance_number': '1', 'model_instance_type': 'Standard', 'model_name': 'dsr1-distill-qwen7fp16b', 'multiNodesInferEnabled': '0', 'multiNodesInferPort': '1120', 'npu_device_id': '1', 'npu_device_ids': '0,1', 'npu_mem': '-1', 'pp': '0', 'rank': '1', 'slaveIPs': '', 'speculation_gamma': '0', 'tp': '0', 'trust_remote_code': '0', 'world_size': '2'}
2025-02-26 15:06:59,126 [INFO] config.py:35 - do not init dmi config
2025-02-26 15:06:59,127 [INFO] standard_model.py:145 - rank id 1 get model config: {'backend_bin_path': '/usr/local/Ascend/mindie/1.0.0/mindie-llm/bin/', 'backend_log_file': '/usr/local/Ascend/mindie/1.0.0/mindie-service/logs/mindservice.log', 'backend_modelInstance_id': '0', 'backend_type': 'atb', 'block_size': '128', 'cpu_mem': '5', 'deploy_type': 'INTER_PROCESS', 'dp': '0', 'ep': '0', 'executor_type': 'LLM_EXECUTOR_PYTHON', 'globalRankIds': '', 'globalWorldSize': '0', 'interNodeKmcKsfMaster': 'tools/pmt/master/ksfa', 'interNodeKmcKsfStandby': 'tools/pmt/standby/ksfb', 'interNodeTLSEnabled': '1', 'interNodeTlsCaFiles': 'ca.pem,', 'interNodeTlsCaPath': 'security/grpc/ca/', 'interNodeTlsCert': 'security/grpc/certs/server.pem', 'interNodeTlsCrlFiles': 'server_crl.pem,', 'interNodeTlsCrlPath': 'security/grpc/certs/', 'interNodeTlsPk': 'security/grpc/keys/server.key.pem', 'interNodeTlsPkPwd': 'security/grpc/pass/mindie_server_key_pwd.txt', 'isMaster': '0', 'localIP': '', 'local_rank': '1', 'log_error': '1', 'log_file_num': '20', 'log_file_size': '20', 'log_info': '1', 'log_verbose': '0', 'log_warning': '1', 'masterIP': '', 'max_input_len': '2048', 'max_iter_times': '512', 'max_prefill_tokens': '8192', 'max_seq_len': '2560', 'model_cut_policy': 'standard', 'model_id': '/data/ollama-data/hf-mirror/Qwen2.5-32B-Instruct-W8A8/', 'model_instance_number': '1', 'model_instance_type': 'Standard', 'model_name': 'dsr1-distill-qwen7fp16b', 'multiNodesInferEnabled': '0', 'multiNodesInferPort': '1120', 'npu_device_id': '1', 'npu_device_ids': '0,1', 'npu_mem': '-1', 'pp': '0', 'rank': '1', 'slaveIPs': '', 'speculation_gamma': '0', 'tp': '0', 'trust_remote_code': '0', 'world_size': '2'}
2025-02-26 15:06:59,269 [INFO] config.py:30 - model_config {'backend_bin_path': '/usr/local/Ascend/mindie/1.0.0/mindie-llm/bin/', 'backend_log_file': '/usr/local/Ascend/mindie/1.0.0/mindie-service/logs/mindservice.log', 'backend_modelInstance_id': '0', 'backend_type': 'atb', 'block_size': '128', 'cpu_mem': '5', 'deploy_type': 'INTER_PROCESS', 'dp': '0', 'ep': '0', 'executor_type': 'LLM_EXECUTOR_PYTHON', 'globalRankIds': '', 'globalWorldSize': '0', 'interNodeKmcKsfMaster': 'tools/pmt/master/ksfa', 'interNodeKmcKsfStandby': 'tools/pmt/standby/ksfb', 'interNodeTLSEnabled': '1', 'interNodeTlsCaFiles': 'ca.pem,', 'interNodeTlsCaPath': 'security/grpc/ca/', 'interNodeTlsCert': 'security/grpc/certs/server.pem', 'interNodeTlsCrlFiles': 'server_crl.pem,', 'interNodeTlsCrlPath': 'security/grpc/certs/', 'interNodeTlsPk': 'security/grpc/keys/server.key.pem', 'interNodeTlsPkPwd': 'security/grpc/pass/mindie_server_key_pwd.txt', 'isMaster': '0', 'localIP': '', 'local_rank': '0', 'log_error': '1', 'log_file_num': '20', 'log_file_size': '20', 'log_info': '1', 'log_verbose': '0', 'log_warning': '1', 'masterIP': '', 'max_input_len': '2048', 'max_iter_times': '512', 'max_prefill_tokens': '8192', 'max_seq_len': '2560', 'model_cut_policy': 'standard', 'model_id': '/data/ollama-data/hf-mirror/Qwen2.5-32B-Instruct-W8A8/', 'model_instance_number': '1', 'model_instance_type': 'Standard', 'model_name': 'dsr1-distill-qwen7fp16b', 'multiNodesInferEnabled': '0', 'multiNodesInferPort': '1120', 'npu_device_id': '0', 'npu_device_ids': '0,1', 'npu_mem': '-1', 'pp': '0', 'rank': '0', 'slaveIPs': '', 'speculation_gamma': '0', 'tp': '0', 'trust_remote_code': '0', 'world_size': '2'}
2025-02-26 15:06:59,269 [INFO] config.py:35 - do not init dmi config
2025-02-26 15:06:59,270 [INFO] standard_model.py:145 - rank id 0 get model config: {'backend_bin_path': '/usr/local/Ascend/mindie/1.0.0/mindie-llm/bin/', 'backend_log_file': '/usr/local/Ascend/mindie/1.0.0/mindie-service/logs/mindservice.log', 'backend_modelInstance_id': '0', 'backend_type': 'atb', 'block_size': '128', 'cpu_mem': '5', 'deploy_type': 'INTER_PROCESS', 'dp': '0', 'ep': '0', 'executor_type': 'LLM_EXECUTOR_PYTHON', 'globalRankIds': '', 'globalWorldSize': '0', 'interNodeKmcKsfMaster': 'tools/pmt/master/ksfa', 'interNodeKmcKsfStandby': 'tools/pmt/standby/ksfb', 'interNodeTLSEnabled': '1', 'interNodeTlsCaFiles': 'ca.pem,', 'interNodeTlsCaPath': 'security/grpc/ca/', 'interNodeTlsCert': 'security/grpc/certs/server.pem', 'interNodeTlsCrlFiles': 'server_crl.pem,', 'interNodeTlsCrlPath': 'security/grpc/certs/', 'interNodeTlsPk': 'security/grpc/keys/server.key.pem', 'interNodeTlsPkPwd': 'security/grpc/pass/mindie_server_key_pwd.txt', 'isMaster': '0', 'localIP': '', 'local_rank': '0', 'log_error': '1', 'log_file_num': '20', 'log_file_size': '20', 'log_info': '1', 'log_verbose': '0', 'log_warning': '1', 'masterIP': '', 'max_input_len': '2048', 'max_iter_times': '512', 'max_prefill_tokens': '8192', 'max_seq_len': '2560', 'model_cut_policy': 'standard', 'model_id': '/data/ollama-data/hf-mirror/Qwen2.5-32B-Instruct-W8A8/', 'model_instance_number': '1', 'model_instance_type': 'Standard', 'model_name': 'dsr1-distill-qwen7fp16b', 'multiNodesInferEnabled': '0', 'multiNodesInferPort': '1120', 'npu_device_id': '0', 'npu_device_ids': '0,1', 'npu_mem': '-1', 'pp': '0', 'rank': '0', 'slaveIPs': '', 'speculation_gamma': '0', 'tp': '0', 'trust_remote_code': '0', 'world_size': '2'}
[2025-02-26 15:07:05,283] [3039769] [140267198744128] [llm] [INFO] [cpu_binding.py-212] : rank_id: 1, device_id: 1, numa_id: 1, shard_devices: [1, 2, 5, 6], cpus: [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95]
[2025-02-26 15:07:05,286] [3039769] [140267198744128] [llm] [INFO] [cpu_binding.py-238] : process 3039769, new_affinity is [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], cpu count 12
[2025-02-26 15:07:05,479] [3039768] [140447560721984] [llm] [INFO] [cpu_binding.py-212] : rank_id: 0, device_id: 0, numa_id: 0, shard_devices: [0, 3, 4, 7], cpus: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71]
[2025-02-26 15:07:05,482] [3039768] [140447560721984] [llm] [INFO] [cpu_binding.py-238] : process 3039768, new_affinity is [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], cpu count 12
[2025-02-26 15:07:05,875] [3039768] [140447560721984] [llm] [INFO] [logging.py-331] : model_runner.quantize: w8a8, model_runner.kv_quant_type: None, model_runner.fa_quant_type: None, model_runner.dtype: torch.float16
[2025-02-26 15:07:09,228] [3039769] [140267198744128] [llm] [INFO] [dist.py-77] : initialize_distributed has been Set
/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py:658: UserWarning: You are using a Backend <class 'atb_llm.utils.dist.FakeGroup'> as a ProcessGroup. This usage is deprecated since PyTorch 2.0. Please use a public API of PyTorch Distributed instead.
  warnings.warn(
[2025-02-26 15:07:09,277] [3039768] [140447560721984] [llm] [INFO] [dist.py-77] : initialize_distributed has been Set
[2025-02-26 15:07:09,278] [3039768] [140447560721984] [llm] [INFO] [logging.py-331] : init tokenizer done: Qwen2TokenizerFast(name_or_path='/data/ollama-data/hf-mirror/Qwen2.5-32B-Instruct-W8A8', vocab_size=151643, model_max_length=131072, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'eos_token': '<|im_end|>', 'pad_token': '<|endoftext|>', 'additional_special_tokens': ['<|im_start|>', '<|im_end|>', '<|object_ref_start|>', '<|object_ref_end|>', '<|box_start|>', '<|box_end|>', '<|quad_start|>', '<|quad_end|>', '<|vision_start|>', '<|vision_end|>', '<|vision_pad|>', '<|image_pad|>', '<|video_pad|>']}, clean_up_tokenization_spaces=False),  added_tokens_decoder={
        151643: AddedToken("<|endoftext|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151644: AddedToken("<|im_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151645: AddedToken("<|im_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151646: AddedToken("<|object_ref_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151647: AddedToken("<|object_ref_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151648: AddedToken("<|box_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151649: AddedToken("<|box_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151650: AddedToken("<|quad_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151651: AddedToken("<|quad_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151652: AddedToken("<|vision_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151653: AddedToken("<|vision_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151654: AddedToken("<|vision_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151655: AddedToken("<|image_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151656: AddedToken("<|video_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151657: AddedToken("<tool_call>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151658: AddedToken("</tool_call>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151659: AddedToken("<|fim_prefix|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151660: AddedToken("<|fim_middle|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151661: AddedToken("<|fim_suffix|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151662: AddedToken("<|fim_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151663: AddedToken("<|repo_name|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
        151664: AddedToken("<|file_sep|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
}
/usr/local/lib/python3.10/dist-packages/torch/distributed/distributed_c10d.py:658: UserWarning: You are using a Backend <class 'atb_llm.utils.dist.FakeGroup'> as a ProcessGroup. This usage is deprecated since PyTorch 2.0. Please use a public API of PyTorch Distributed instead.
  warnings.warn(
[2025-02-26 15:07:09,288] [3039768] [140447560721984] [llm] [INFO] [logging.py-331] : NPUSocInfo(soc_name='', soc_version=103, need_nz=True, matmul_nd_nz=False)
2025-02-26 15:07:09,347 [ERROR] model.py:39 - [Model]   >>> Exception:call aclnnInplaceZero failed, detail:EZ9999: Inner Error!
EZ9999: [PID: 3039769] 2025-02-26-15:07:09.339.604 Parse dynamic kernel config fail.
        TraceBack (most recent call last):
       AclOpKernelInit failed opType
       ZerosLike ADD_TO_LAUNCHER_LIST_AICORE failed.

[ERROR] 2025-02-26-15:07:09 (PID:3039769, Device:1, RankID:-1) ERR01100 OPS call acl api failed
Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/model_wrapper/model.py", line 37, in initialize
    return self.python_model.initialize(config)
  File "/usr/local/lib/python3.10/dist-packages/model_wrapper/standard_model.py", line 146, in initialize
    self.generator = Generator(
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/text_generator/generator.py", line 79, in __init__
    self.generator_backend = get_generator_backend(model_config)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/text_generator/adapter/__init__.py", line 26, in get_generator_backend
    return generator_cls(model_config)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 96, in __init__
    super().__init__(model_config)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/text_generator/adapter/generator_backend.py", line 107, in __init__
    self.model_wrapper = get_model_wrapper(model_config, backend_type)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/modeling/model_wrapper/__init__.py", line 15, in get_model_wrapper
    return wrapper_cls(**model_config)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 46, in __init__
    self.model_runner.load_weights()
  File "/data/ollama-data/atlas300/Ascend-mindie-atb-models_1.0.0_linux-x86_64_py310_torch2.3.1-abi0/atb_llm/runner/model_runner.py", line 149, in load_weights
    self.model = self.model_cls(self.config, weights, inference_mode=self.inference_mode,
  File "/data/ollama-data/atlas300/Ascend-mindie-atb-models_1.0.0_linux-x86_64_py310_torch2.3.1-abi0/atb_llm/models/qwen2/flash_causal_qwen2.py", line 24, in __init__
    super().__init__(config, weights, **kwargs)
  File "/data/ollama-data/atlas300/Ascend-mindie-atb-models_1.0.0_linux-x86_64_py310_torch2.3.1-abi0/atb_llm/models/base/flash_causal_lm.py", line 98, in __init__
    self.placeholder = torch.zeros(1, dtype=self.dtype, device="npu")
RuntimeError: call aclnnInplaceZero failed, detail:EZ9999: Inner Error!
EZ9999: [PID: 3039769] 2025-02-26-15:07:09.339.604 Parse dynamic kernel config fail.
        TraceBack (most recent call last):
       AclOpKernelInit failed opType
       ZerosLike ADD_TO_LAUNCHER_LIST_AICORE failed.

[ERROR] 2025-02-26-15:07:09 (PID:3039769, Device:1, RankID:-1) ERR01100 OPS call acl api failed
2025-02-26 15:07:09,348 [ERROR] model.py:42 - [Model]   >>> return initialize error result: {'status': 'error', 'npuBlockNum': '0', 'cpuBlockNum': '0'}
2025-02-26 15:07:09,355 [ERROR] model.py:39 - [Model]   >>> Exception:call aclnnInplaceZero failed, detail:EZ9999: Inner Error!
EZ9999: [PID: 3039768] 2025-02-26-15:07:09.348.197 Parse dynamic kernel config fail.
        TraceBack (most recent call last):
       AclOpKernelInit failed opType
       ZerosLike ADD_TO_LAUNCHER_LIST_AICORE failed.

[ERROR] 2025-02-26-15:07:09 (PID:3039768, Device:0, RankID:-1) ERR01100 OPS call acl api failed
Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/model_wrapper/model.py", line 37, in initialize
    return self.python_model.initialize(config)
  File "/usr/local/lib/python3.10/dist-packages/model_wrapper/standard_model.py", line 146, in initialize
    self.generator = Generator(
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/text_generator/generator.py", line 79, in __init__
    self.generator_backend = get_generator_backend(model_config)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/text_generator/adapter/__init__.py", line 26, in get_generator_backend
    return generator_cls(model_config)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 96, in __init__
    super().__init__(model_config)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/text_generator/adapter/generator_backend.py", line 107, in __init__
    self.model_wrapper = get_model_wrapper(model_config, backend_type)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/modeling/model_wrapper/__init__.py", line 15, in get_model_wrapper
    return wrapper_cls(**model_config)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 46, in __init__
    self.model_runner.load_weights()
  File "/data/ollama-data/atlas300/Ascend-mindie-atb-models_1.0.0_linux-x86_64_py310_torch2.3.1-abi0/atb_llm/runner/model_runner.py", line 149, in load_weights
    self.model = self.model_cls(self.config, weights, inference_mode=self.inference_mode,
  File "/data/ollama-data/atlas300/Ascend-mindie-atb-models_1.0.0_linux-x86_64_py310_torch2.3.1-abi0/atb_llm/models/qwen2/flash_causal_qwen2.py", line 24, in __init__
    super().__init__(config, weights, **kwargs)
  File "/data/ollama-data/atlas300/Ascend-mindie-atb-models_1.0.0_linux-x86_64_py310_torch2.3.1-abi0/atb_llm/models/base/flash_causal_lm.py", line 98, in __init__
    self.placeholder = torch.zeros(1, dtype=self.dtype, device="npu")
RuntimeError: call aclnnInplaceZero failed, detail:EZ9999: Inner Error!
EZ9999: [PID: 3039768] 2025-02-26-15:07:09.348.197 Parse dynamic kernel config fail.
        TraceBack (most recent call last):
       AclOpKernelInit failed opType
       ZerosLike ADD_TO_LAUNCHER_LIST_AICORE failed.

[ERROR] 2025-02-26-15:07:09 (PID:3039768, Device:0, RankID:-1) ERR01100 OPS call acl api failed
2025-02-26 15:07:09,356 [ERROR] model.py:42 - [Model]   >>> return initialize error result: {'status': 'error', 'npuBlockNum': '0', 'cpuBlockNum': '0'}
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
Process ForkServerProcess-9:
Traceback (most recent call last):
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 810, in _callmethod
    conn = self._tls.connection
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
    raise exp
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
    func(*args, **kwargs)
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
    resource_proxy[SUB_PROCESS_STATE].append(True)
  File "<string>", line 2, in append
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 814, in _callmethod
    self._connect()
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 801, in _connect
    conn = self._Client(self._token.address, authkey=self._authkey)
  File "/usr/lib/python3.10/multiprocessing/connection.py", line 502, in Client
    c = SocketClient(address)
  File "/usr/lib/python3.10/multiprocessing/connection.py", line 630, in SocketClient
    s.connect(address)
ConnectionRefusedError: [Errno 111] Connection refused
/usr/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 30 leaked semaphore objects to clean up at shutdown
  warnings.warn('resource_tracker: There appear to be %d '
/usr/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 30 leaked semaphore objects to clean up at shutdown
  warnings.warn('resource_tracker: There appear to be %d '
Daemon is killing...
Killed

2025-02-26 15:18:28,056 [ERROR] model.py:39 - [Model]   >>> Exception:call aclnnInplaceZero failed, detail:EZ9999: Inner Error!
EZ9999: [PID: 3069016] 2025-02-26-15:18:28.050.204 Parse dynamic kernel config fail.
        TraceBack (most recent call last):
       AclOpKernelInit failed opType
       ZerosLike ADD_TO_LAUNCHER_LIST_AICORE failed.

[ERROR] 2025-02-26-15:18:28 (PID:3069016, Device:0, RankID:-1) ERR01100 OPS call acl api failed
Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/model_wrapper/model.py", line 37, in initialize
    return self.python_model.initialize(config)
  File "/usr/local/lib/python3.10/dist-packages/model_wrapper/standard_model.py", line 146, in initialize
    self.generator = Generator(
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/text_generator/generator.py", line 79, in __init__
    self.generator_backend = get_generator_backend(model_config)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/text_generator/adapter/__init__.py", line 26, in get_generator_backend
    return generator_cls(model_config)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 96, in __init__
    super().__init__(model_config)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/text_generator/adapter/generator_backend.py", line 107, in __init__
    self.model_wrapper = get_model_wrapper(model_config, backend_type)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/modeling/model_wrapper/__init__.py", line 15, in get_model_wrapper
    return wrapper_cls(**model_config)
  File "/usr/local/lib/python3.10/dist-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 46, in __init__
    self.model_runner.load_weights()
  File "/data/ollama-data/atlas300/Ascend-mindie-atb-models_1.0.0_linux-x86_64_py310_torch2.3.1-abi0/atb_llm/runner/model_runner.py", line 149, in load_weights
    self.model = self.model_cls(self.config, weights, inference_mode=self.inference_mode,
  File "/data/ollama-data/atlas300/Ascend-mindie-atb-models_1.0.0_linux-x86_64_py310_torch2.3.1-abi0/atb_llm/models/qwen2/flash_causal_qwen2.py", line 24, in __init__
    super().__init__(config, weights, **kwargs)
  File "/data/ollama-data/atlas300/Ascend-mindie-atb-models_1.0.0_linux-x86_64_py310_torch2.3.1-abi0/atb_llm/models/base/flash_causal_lm.py", line 98, in __init__
    self.placeholder = torch.zeros(1, dtype=self.dtype, device="npu")
RuntimeError: call aclnnInplaceZero failed, detail:EZ9999: Inner Error!
EZ9999: [PID: 3069016] 2025-02-26-15:18:28.050.204 Parse dynamic kernel config fail.
        TraceBack (most recent call last):
       AclOpKernelInit failed opType
       ZerosLike ADD_TO_LAUNCHER_LIST_AICORE failed
       
       
       
[ERROR] 2025-02-26-15:18:28 (PID:3069017, Device:1, RankID:-1) ERR01100 OPS call acl api failed
2025-02-26 15:18:28,150 [ERROR] model.py:42 - [Model]   >>> return initialize error result: {'status': 'error', 'npuBlockNum': '0', 'cpuBlockNum': '0'}
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
Process ForkServerProcess-9:
Process ForkServerProcess-3:
Traceback (most recent call last):
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 810, in _callmethod
    conn = self._tls.connection
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
    raise exp
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
    func(*args, **kwargs)
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
    resource_proxy[SUB_PROCESS_STATE].append(True)
  File "<string>", line 2, in append
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 814, in _callmethod
    self._connect()
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 801, in _connect
    conn = self._Client(self._token.address, authkey=self._authkey)
  File "/usr/lib/python3.10/multiprocessing/connection.py", line 502, in Client
    c = SocketClient(address)
  File "/usr/lib/python3.10/multiprocessing/connection.py", line 630, in SocketClient
    s.connect(address)
ConnectionRefusedError: [Errno 111] Connection refused
Traceback (most recent call last):
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 810, in _callmethod
    conn = self._tls.connection
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
    raise exp
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
    func(*args, **kwargs)
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
    resource_proxy[SUB_PROCESS_STATE].append(True)
  File "<string>", line 2, in append
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 814, in _callmethod
    self._connect()
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 801, in _connect
    conn = self._Client(self._token.address, authkey=self._authkey)
  File "/usr/lib/python3.10/multiprocessing/connection.py", line 502, in Client
    c = SocketClient(address)
  File "/usr/lib/python3.10/multiprocessing/connection.py", line 630, in SocketClient
    s.connect(address)
ConnectionRefusedError: [Errno 111] Connection refused
Process ForkServerProcess-9:
Process ForkServerProcess-4:
Process ForkServerProcess-6:
Process ForkServerProcess-8:
Process ForkServerProcess-3:
Process ForkServerProcess-5:
Process ForkServerProcess-2:
Process ForkServerProcess-7:
Traceback (most recent call last):
Traceback (most recent call last):
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 810, in _callmethod
    conn = self._tls.connection
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 810, in _callmethod
    conn = self._tls.connection
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 810, in _callmethod
    conn = self._tls.connection
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 810, in _callmethod
    conn = self._tls.connection
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 810, in _callmethod
    conn = self._tls.connection
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 810, in _callmethod
    conn = self._tls.connection
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 810, in _callmethod
    conn = self._tls.connection
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'

During handling of the above exception, another exception occurred:

AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
Traceback (most recent call last):
  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'

During handling of the above exception, another exception occurred:

  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
    self.run()

During handling of the above exception, another exception occurred:

  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)

During handling of the above exception, another exception occurred:


During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
Traceback (most recent call last):
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
    raise exp
  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
    raise exp
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
    func(*args, **kwargs)
  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
    func(*args, **kwargs)
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 810, in _callmethod
    conn = self._tls.connection
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
    resource_proxy[SUB_PROCESS_STATE].append(True)
  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
    raise exp
  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
    resource_proxy[SUB_PROCESS_STATE].append(True)
  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "<string>", line 2, in append
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
    func(*args, **kwargs)
  File "/usr/local/Ascend/nnae/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
    raise exp
  File "<string>", line 2, in append
  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.10/multiprocessing/managers.py", line 814, in _callmethod
    self._connect()

[2025-02-26 14:38:08.398925] [error] [2899194] [rt_backend_module.cpp:180] rt KernelLaunch With Handle fail, error:207001
[2025-02-26 14:38:08.399637] [error] [2899194] [kernel_base.cpp:307] Mki RtFunction LaunchWithHandle fail
[2025-02-26 14:38:08.399644] [error] [2899194] [mki_node_implement.cpp:211] LinearOpsRunner_32_1_0_0[2] TransdataNzToNdKernel run fail, launchParam
:specificParam: {"outCrops":[4,75968],"transdataType":1}
intensors[0]: dtype:float16, format:fractal_nz, dims:[1, 4748, 16, 16], dataSize:2430976, data:1
outtensors[0]: dtype:float16, format:nd, dims:[1, 4, 75968], dataSize:607744, data:1


st: code:7, msg:Mki RtFunction LaunchWithHandle fail
[2025-02-26 14:38:08.399680] [error] [2899194] [ops_runner.cpp:602] RunKernel failed! ret:3
[2025-02-26 14:38:08.399683] [error] [2899194] [ops_runner.cpp:397] RunAllKernel failed! ret:3
[2025-02-26 14:38:08.399687] [error] [2899194] [runner.cpp:115] LinearOpsRunner_32_1_0_0:1 Execute Failed. st: 3
[2025-02-26 14:38:08.399693] [error] [2899194] [graph_runner.cpp:906] LinearNoQuantRunner_32_1_0:0  node[0] execute fail, runner name:LinearOpsRunner
[2025-02-26 14:38:08.399698] [error] [2899194] [runner.cpp:115] LinearNoQuantRunner_32_1_0:1 Execute Failed. st: 3
[2025-02-26 14:38:08.399702] [error] [2899194] [graph_runner.cpp:906] LinearColumnParallelRunner_32_1:0  node[0] execute fail, runner name:LinearNoQuantRunner
[2025-02-26 14:38:08.399706] [error] [2899194] [runner.cpp:115] LinearColumnParallelRunner_32_1:1 Execute Failed. st: 3
[2025-02-26 14:38:08.399709] [error] [2899194] [graph_runner.cpp:906] LmHeadRunner_32:0  node[1] execute fail, runner name:LinearColumnParallelRunner
[2025-02-26 14:38:08.399714] [error] [2899194] [runner.cpp:115] LmHeadRunner_32:1 Execute Failed. st: 3
[2025-02-26 14:38:08.399718] [error] [2899194] [operation_base.cpp:726] LmHead_32 execute LmHeadRunner fail
[2025-02-26 14:38:08.399721] [error] [2899194] [operation_base.cpp:764] LmHead_32 execute fail, error code: 3