export ATB_LLM_HCCL_ENABLE=1
export ATB_LLM_COMM_BACKEND="hccl"
export HCCL_CONNECT_TIMEOUT=7200
export WORLD_SIZE=16
export HCCL_EXEC_TIMEOUT=0
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export MIES_CONTAINER_IP=172.16.144.144
export RANKTABLEFILE=/mnt/zju2/code_aaron/rank_table_file.json


# 以下命令需在所有机器上同时执行
# 解决权重加载过慢问题
export OMP_NUM_THREADS=1
# 设置显存比
export NPU_MEMORY_FRACTION=0.96

File "/usr/local/Ascend/atb-models/atb_llm/models/deepseekv2/flash_causal_deepseekv2.py", line 218, in get_weights
    weight_wrapper.register_moe_layer(layer, self.quantize, dense_layer=False,
  File "/usr/local/Ascend/atb-models/atb_llm/utils/data/moe_weight_wrapper.py", line 309, in register_moe_layer
    self.register_layer_moe_mlp_experts(layer, self.moe_mlp_wrapper, quantize_type, expert_roster)
  File "/usr/local/Ascend/atb-models/atb_llm/utils/data/moe_weight_wrapper.py", line 249, in register_layer_moe_mlp_experts
    self.register_layer_moe_linear_pack(layer, wrapper, quantize_type, 'moe_mlp', expert_roster)
  File "/usr/local/Ascend/atb-models/atb_llm/utils/data/moe_weight_wrapper.py", line 207, in register_layer_moe_linear_pack
    self.register_linear_list_wrapper(linear_list, quantize_type, hidden_dim=wrapper_module.hidden_dim)
  File "/usr/local/Ascend/atb-models/atb_llm/utils/data/moe_weight_wrapper.py", line 126, in register_linear_list_wrapper
    trans_flag = self.register_linear_list_bias(linear_list, quantize_type,
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/Ascend/atb-models/atb_llm/utils/data/moe_weight_wrapper.py", line 77, in register_linear_list_bias
    trans_flag = self.register_linear_list_without_bias(linear_list, quantize_type,
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/Ascend/atb-models/atb_llm/utils/data/moe_weight_wrapper.py", line 65, in register_linear_list_without_bias
    tensor_stacked = torch.stack([linear.weight.data.transpose(0, 1) for linear in linear_list], dim=0)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: NPU out of memory. Tried to allocate 506.00 MiB (NPU 3; 60.00 GiB total capacity; 45.65 GiB already allocated; 45.65 GiB current active; 92.09 MiB free; 51.65 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.

[WARN]operator(),compiler_depend.ts:732:Feature is not supportted and the possible cause is that driver and firmware packages do[WARN]operator(),compiler_depend.ts:732:Feature is not supportted and the possible cause is that driver and firmware packages do not match.
 not match.
[WARN]operator(),compiler_depend.ts:732:Feature is not supportted and the possible cause is that driver and firmware packages do not match.
[WARN]operator(),compiler_depend.ts:732:Feature is not supportted and the possible cause is that driver and firmware packages do not match.
[W compiler_depend.ts:733] Warning: expandable_segments setting failure, now change to `False`. (function operator())
[WARN]operator(),compiler_depend.ts:732:Feature is not supportted and the possible cause is that driver and firmware packages do not match.
[WARN]operator(),compiler_depend.ts:732:Feature is not supportted and the possible cause is that driver and firmware packages do not match.
[WARN]operator(),compiler_depend.ts:732:Feature is not supportted and the possible cause is that driver and firmware packages do not match.
[WARN]operator(),compiler_depend.ts:732:Feature is not supportted and the possible cause is that driver and firmware packages do not match.