Lora微调是出现oom 

机器 

8卡 NPU显存64G 

报错信息

Traceback (most recent call last): 

  File "pretrain_gpt.py", line 280, in <module> 

    main() 

  File "pretrain_gpt.py", line 273, in main 

    pretrain(train_valid_test_datasets_provider, 

  File "/home/leo/ModelLink/modellink/training/training.py", line 323, in pretrain 

    train_args, test_data_iterator_list = build_train_args(args, timers, train_valid_test_dataset_provider, 

  File "/home/leo/ModelLink/modellink/training/training.py", line 210, in build_train_args 

    model, optimizer, opt_param_scheduler = setup_model_and_optimizer( 

  File "/home/leo/ModelLink/modellink/training/training.py", line 195, in wrapper 

    model, optimizer, opt_param_scheduler = fn(*args, **kwargs) 

  File "/home/leo/ModelLink/megatron/training/training.py", line 502, in setup_model_and_optimizer 

    optimizer = get_megatron_optimizer(config, model, no_wd_decay_cond, 

  File "/home/leo/ModelLink/modellink/core/optimizer/__init__.py", line 252, in wrapper 

    return fn(*args, **kwargs) 

  File "/home/leo/ModelLink/megatron/core/optimizer/__init__.py", line 315, in get_megatron_optimizer 

    _get_megatron_optimizer_based_on_param_groups( 

  File "/home/leo/ModelLink/megatron/core/optimizer/__init__.py", line 247, in _get_megatron_optimizer_based_on_param_groups 

    optimizer = Float16OptimizerWithFloat16Params(*optimizer_args) 

  File "/home/leo/ModelLink/megatron/core/optimizer/optimizer.py", line 425, in __init__ 

    main_param = param.detach().clone().float() 

RuntimeError: NPU out of memory. Tried to allocate 232.00 MiB (NPU 3; 60.97 GiB total capacity; 59.78 GiB already allocated; 59.78 GiB current active; 21.52 MiB free; 60.25 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation. 

 

 运行参数配置

#!/bin/bash 

 

export CUDA_DEVICE_MAX_CONNECTIONS=1 

 

GPUS_PER_NODE=8 

MASTER_ADDR=localhost 

MASTER_PORT=6005 

NNODES=1 

NODE_RANK=0 

WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES)) 

 

CKPT_SAVE_DIR="./ckpt/fine_tune_qwen2_72b_lora_ptd/" 

DATA_PATH="./finetune_dataset/alpaca" 

TOKENIZER_MODEL="./model_from_hf/Qwen2-72B-Instruct/" 

CKPT_LOAD_DIR="./model_weights/qwen2_72b_mcore/" 

LORA_CHECKPOINT="./ckpt/fine_tune_qwen2_72b_lora_ptd/" 

 

TP=8 

PP=1 

 

DISTRIBUTED_ARGS=" 

    --nproc_per_node $GPUS_PER_NODE \ 

    --nnodes $NNODES \ 

    --node_rank $NODE_RANK \ 

    --master_addr $MASTER_ADDR \ 

    --master_port $MASTER_PORT 

" 

GPT_ARGS=" 

    --use-mcore-models \ 

    --tensor-model-parallel-size ${TP} \ 

    --pipeline-model-parallel-size ${PP} \ 

    --variable-seq-lengths \ 

    --load ${CKPT_LOAD_DIR} \ 

    --num-layers 80 \ 

    --hidden-size 8192 \ 

    --ffn-hidden-size 29568 \ 

    --num-attention-heads 64 \ 

    --tokenizer-type PretrainedFromHF \ 

    --tokenizer-name-or-path ${TOKENIZER_MODEL} \ 

    --tokenizer-not-use-fast \ 

    --seq-length 32768 \ 

    --max-position-embeddings 32768 \ 

    --micro-batch-size 1 \ 

    --global-batch-size 4 \ 

    --make-vocab-size-divisible-by 1 \ 

    --train-iters 20 \ 

    --disable-bias-linear \ 

    --attention-dropout 0.0 \ 

    --hidden-dropout 0.0 \ 

    --position-embedding-type rope \ 

    --untie-embeddings-and-output-weights \ 

    --normalization RMSNorm \ 

    --swiglu \ 

    --use-flash-attn \ 

    --use-fused-rmsnorm \ 

    --no-masked-softmax-fusion \ 

    --attention-softmax-in-fp32 \ 

    --lr 5.0e-5 \ 

    --lr-decay-style cosine \ 

    --lr-warmup-fraction 0.01 \ 

    --min-lr 1.25e-6 \ 

    --weight-decay 1e-1 \ 

    --clip-grad 1.0 \ 

    --adam-beta1 0.9 \ 

    --adam-beta2 0.999 \ 

    --initial-loss-scale 1 \ 

    --init-method-std 0.01 \ 

    --no-gradient-accumulation-fusion \ 

    --no-load-optim \ 

    --no-load-rng \ 

    --finetune \ 

    --is-instruction-dataset \ 

    --lora-r 8 \ 

    --lora-alpha 16 \ 

    --lora-fusion \ 

    --rotary-base 10000 \ 

    --norm-epsilon 1e-05 \ 

    --padded-vocab-size 152064 \ 

    --log-throughput \ 

    --add-qkv-bias \ 

    --prompt-type qwen \ 

    --group-query-attention \ 

    --num-query-groups 8 \ 

    --bf16 \ 

" 

DATA_ARGS=" 

    --data-path $DATA_PATH \ 

    --split 100,0,0 

" 

 

OUTPUT_ARGS=" 

    --log-interval 1 \ 

    --save-interval 20 \ 

    --eval-interval 10 \ 

    --eval-iters 0 \ 

" 

 

torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \ 

    $GPT_ARGS \ 

    $DATA_ARGS \ 

    $OUTPUT_ARGS \ 

    --distributed-backend nccl \ 

    --save $CKPT_SAVE_DIR \ 

    | tee logs/tune_qwen2_72b_lora_ptd.log 

 

请问是否是参数配置有误导致的还是就是显存不够。