[rank1]: Traceback (most recent call last):
[rank1]:   File "/data1/guihonghao/ms-swift/swift/cli/sft.py", line 5, in <module>
[rank1]:     sft_main()
[rank1]:   File "/data1/guihonghao/ms-swift/swift/utils/run_utils.py", line 22, in x_main
[rank1]:     args, remaining_argv = parse_args(args_class, argv)
[rank1]:   File "/data1/guihonghao/ms-swift/swift/utils/utils.py", line 131, in parse_args
[rank1]:     args, remaining_args = parser.parse_args_into_dataclasses(argv, return_remaining_strings=True)
[rank1]:   File "/home/guihonghao/.local/lib/python3.10/site-packages/transformers/hf_argparser.py", line 352, in parse_args_into_dataclasses
[rank1]:     obj = dtype(**inputs)
[rank1]:   File "<string>", line 215, in __init__
[rank1]:   File "/data1/guihonghao/ms-swift/swift/llm/utils/argument.py", line 1165, in __post_init__
[rank1]:     self.output_dir = add_version_to_work_dir(self.output_dir)
[rank1]:   File "/data1/guihonghao/ms-swift/swift/utils/utils.py", line 113, in add_version_to_work_dir
[rank1]:     sub_folder = broadcast_string(sub_folder)
[rank1]:   File "/data1/guihonghao/ms-swift/swift/utils/torch_utils.py", line 211, in broadcast_string
[rank1]:     tensor = torch.zeros(buffer_size, dtype=torch.int64, device=device)
[rank1]: RuntimeError: map:build/CMakeFiles/torch_npu.dir/compiler_depend.ts:373 NPU function error: aclrtMallocPhysical, error code is 507899
[rank1]: [ERROR] 2024-11-27-00:33:19 (PID:2941779, Device:1, RankID:1) ERR00100 PTA call acl api failed
[rank1]: [Error]: An internal error occurs in the Driver module. 
[rank1]:         Rectify the fault based on the error information in the ascend log.
[rank1]: EL9999: Inner Error!
[rank1]: EL9999  [drv api]halMemCreate failed. drvRet=17.[FUNC:MallocPhysical][FILE:npu_driver.cc][LINE:5439]
[rank1]:         TraceBack (most recent call last):
[rank1]:         rtMallocPhysical execute failed, reason=[driver error:internal error][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:50]
[rank1]:         malloc physical memory failed, runtime result = 507899[FUNC:ReportCallError][FILE:log_inner.cpp][LINE:161]

source /usr/local/Ascend/ascend-toolkit/set_env.sh
export MODELSCOPE_CACHE='/data1/guihonghao/.cache/modelscope'
export HF_DATASETS_CACHE='/data1/guihonghao/.cache/huggingface'
export NPROC_PER_NODE=4 
export ASCEND_RT_VISIBLE_DEVICES=2,3,4,5 
export ASCEND_LAUNCH_BLOCKING=1 
swift sft \
    --model_type 'qwen2_5-7b-instruct' \
    --model_id_or_path /data1/guihonghao/models/Qwen2.5-7B-Instruct \
    --sft_type 'lora' \
    --tuner_backend 'peft' \
    --template_type 'AUTO' \
    --dtype 'bf16' \
    --output_dir /data1/guihonghao/lora_outputs/qwen2_5-7b-instruct-ie-v3 \
    --custom_train_dataset_path /data2/guihonghao/data/train_data/IE-v1/res/train.jsonl \
    --custom_val_dataset_path /data2/guihonghao/data/train_data/IE-v1/res/dev.jsonl \
    --num_train_epochs 5 \
    --max_length 2048 \
    --check_dataset_strategy 'warning' \
    --lora_rank 64 \
    --lora_alpha 64 \
    --lora_dropout_p 0.05 \
    --lora_target_modules 'DEFAULT' \
    --gradient_checkpointing true \
    --batch_size 6 \
    --eval_batch_size 2 \
    --weight_decay 0.1 \
    --learning_rate '5e-5' \
    --gradient_accumulation_steps 4 \
    --max_grad_norm 0.5 \
    --warmup_ratio 0.03 \
    --eval_strategy 'no' \
    --save_strategy 'epoch' \
    --save_total_limit 10 \
    --logging_steps 10 \
    --preprocess_num_proc 32 \
    --lazy_tokenize True \
    --deepspeed 'default-zero2'