昇腾社区首页
支持
消息
我的论坛主页
论坛首页
/
A3进行YOLOv8-obb超大分辨率训练显存报错
收藏回复举报
A3进行YOLOv8-obb超大分辨率训练显存报错
t('forum.solved') 已解决
发表于2026-03-04 15:25:04
0 查看

HDK: 25.0.rc1.3

CANN: 8.3.RC1

代码：

url=https://gitee.com/ascend/modelzoo-GPL

code_path=built-in/PyTorch/Official/cv/object_detection

测试脚本：

#!/bin/bash

#网络名称,同目录名称,需要模型审视修改
Network="yolov8_ID8340_for_PyTorch"

cur_path=`pwd`
batch_size=16
epochs=1
RANK_SIZE=16
OUT_DIR_NAME="full_16p"

for para in $*
do
   if [[ $para == --batch_size* ]];then
      batch_size=`echo ${para#*=}`
   elif [[ $para == --epochs* ]];then
      epochs=`echo ${para#*=}`
   fi
done

ASCEND_DEVICE_ID=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
export ASCEND_RT_VISIBLE_DEVICES=$ASCEND_DEVICE_ID

# 终极显存优化配置（新增更多关键项）
export HCCL_IF_IP=$(hostname -I | awk '{print $1}')
export MASTER_PORT=$((RANDOM % 10000 + 20000))
export HCCL_PORT_REUSE_ENABLE=1

# ========== 昇腾A3 16die 超大分辨率专属显存优化（核心新增） ==========
export ASCEND_NPU_DVPP_TYPE=1                     # A3专属DVPP加速，降低图片解码显存
export ASCEND_MEMORY_POOL_TYPE=1                  # 16die分布式显存池调度
export ASCEND_MEMORY_POOL_SIZE=0                  # 自动适配16die显存池大小
export ASCEND_SINGLE_DEVICE_MEMORY_SIZE=32        # 单卡显存上限32GB（A3单卡显存）
export ASCEND_LARGE_MODEL_OPT=3                   # A3超大模型深度优化（比1/2更适配16die）
export ASCEND_FP16_MODE=1                         # 强制FP16计算（显存减半，精度损失<1%）
export ASCEND_OP_PRECISION_MODE=force_fp16        # 算子级FP16，进一步降显存
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True,memory_limit=0.9  # 显存使用率限制90%
export TASK_QUEUE_ENABLE=2                        # A3专属任务队列（比1更适配16die）
export ASCEND_GLOBAL_MEM_POOL_ENABLE=2            # 全局显存池（跨die复用）
export ASCEND_DISABLE_HCCL_P2P=1                  # 关闭P2P，避免跨die显存占用
export ASCEND_COMPILER_CACHE_MODE=1               # 算子缓存，减少编译显存占用
export ASCEND_EAGER_MODE=1                        # 即时执行模式，降低显存峰值
# ========== 原有配置保留 ==========
export HCCL_IF_IP=$(hostname -I | awk '{print $1}')
export MASTER_PORT=$((RANDOM % 10000 + 20000))
export HCCL_PORT_REUSE_ENABLE=1
export PYTHONWARNINGS="ignore"


#创建DeviceID输出目录，不需要修改
if [ -d ${cur_path}/test/output/${OUT_DIR_NAME} ];
        then
           rm -rf ${cur_path}/test/output/${OUT_DIR_NAME}
                mkdir -p ${cur_path}/test/output/${OUT_DIR_NAME}
        else
           mkdir -p ${cur_path}/test/output/${OUT_DIR_NAME}
        fi

#训练开始时间，不需要修改
start_time=$(date +%s)
echo "start_time: ${start_time}"

source ${cur_path}/test/env_npu.sh

python3 -u train.py --data ./ultralytics/cfg/datasets/DOTAv2.0.yaml \
                     --cfg ./ultralytics/cfg/models/v8/yolov8-obb.yaml \
                     --weights ./yolov8n-obb.pt \
                     --batch $batch_size \
                     --data_shuffle \
                     --imgsz 12000 \
                     --device $ASCEND_DEVICE_ID \
                     --epochs $epochs > $cur_path/test/output/${OUT_DIR_NAME}/train_16p.log 2>&1 &

wait

# #训练结束时间，不需要修改
end_time=$(date +%s)
echo "end_time: ${end_time}"
e2e_time=$(( $end_time - $start_time ))

# 计算FPS的平均值
total_FPS=`grep -oP 'FPS:\K\s*(\d+\.?\d*)' ${cur_path}/test/output/$OUT_DIR_NAME/train_16p.log | tail -n 78 | awk '{sum += $1} END {print sum}'`
averageFPS=$(echo "scale=2; $total_FPS/78" | bc)

# 取mAP50的值
mAP50=$(grep -w 'all' ${cur_path}/test/output/$OUT_DIR_NAME/train_16p.log | awk -F "all" '{print $2}' | awk -F "    " '{print $7}' |  tail -n1)
# 取mAP50-95的值
mAP50_95=$(grep -w 'all' ${cur_path}/test/output/$OUT_DIR_NAME/train_16p.log | awk -F "all" '{print $2}' | awk -F "    " '{print $8}' |  tail -n1)

#打印，不需要修改
echo "Average Performance images/sec : $averageFPS"
echo "mAP50 : $mAP50"
echo "mAP50-95 : $mAP50_95"
echo "E2E Training Duration sec : $e2e_time"

#稳定性精度看护结果汇总
#训练用例信息，不需要修改
BatchSize=${batch_size}
DeviceType=`uname -m`
CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'acc'

##获取性能数据，不需要修改
#单迭代训练时长
TrainingTime=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'*1000/'${averageFPS}'}'`

#关键信息打印到${CaseName}.log中，不需要修改
echo "Network = ${Network}" > $cur_path/test/output/$OUT_DIR_NAME/${CaseName}.log
echo "RankSize = ${RANK_SIZE}" >> $cur_path/test/output/$OUT_DIR_NAME/${CaseName}.log
echo "BatchSize = ${BatchSize}" >> $cur_path/test/output/$OUT_DIR_NAME/${CaseName}.log
echo "DeviceType = ${DeviceType}" >> $cur_path/test/output/$OUT_DIR_NAME/${CaseName}.log
echo "CaseName = ${CaseName}" >> $cur_path/test/output/$OUT_DIR_NAME/${CaseName}.log
echo "averageFPS = ${averageFPS}" >> $cur_path/test/output/$OUT_DIR_NAME/${CaseName}.log
echo "mAP50 = ${mAP50}" >> $cur_path/test/output/$OUT_DIR_NAME/${CaseName}.log
echo "mAP50-95 = ${mAP50_95}" >> $cur_path/test/output/$OUT_DIR_NAME/${CaseName}.log
echo "TrainingTime = ${TrainingTime}" >> $cur_path/test/output/$OUT_DIR_NAME/${CaseName}.log
echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/test/output/$OUT_DIR_NAME/${CaseName}.log

关键报错日志：

tail -f /data/yolov8-train-datav1/modelzoo-GPL/built-in/PyTorch/Official/cv/object_detection/Yolov8_for_PyTorch/test/output/full_16p/train_16p.log
/usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/contrib/transfer_to_npu.py:311: ImportWarning: 
    *************************************************************************************************************
    The torch.Tensor.cuda and torch.nn.Module.cuda are replaced with torch.Tensor.npu and torch.nn.Module.npu now..
    The torch.cuda.DoubleTensor is replaced with torch.npu.FloatTensor cause the double type is not supported now..
    The backend in torch.distributed.init_process_group set to hccl now..
    The torch.cuda.* and torch.cuda.amp.* are replaced with torch.npu.* and torch.npu.amp.* now..
    The device parameters have been replaced with npu in the function below:
    torch.logspace, torch.randint, torch.hann_window, torch.rand, torch.full_like, torch.ones_like, torch.rand_like, torch.randperm, torch.arange, torch.frombuffer, torch.normal, torch._empty_per_channel_affine_quantized, torch.empty_strided, torch.empty_like, torch.scalar_tensor, torch.tril_indices, torch.bartlett_window, torch.ones, torch.sparse_coo_tensor, torch.randn, torch.kaiser_window, torch.tensor, torch.triu_indices, torch.as_tensor, torch.zeros, torch.randint_like, torch.full, torch.eye, torch._sparse_csr_tensor_unsafe, torch.empty, torch._sparse_coo_tensor_unsafe, torch.blackman_window, torch.zeros_like, torch.range, torch.sparse_csr_tensor, torch.randn_like, torch.from_file, torch._cudnn_init_dropout_state, torch._empty_affine_quantized, torch.linspace, torch.hamming_window, torch.empty_quantized, torch._pin_memory, torch.load, torch.set_default_device, torch.Tensor.new_empty, torch.Tensor.new_empty_strided, torch.Tensor.new_full, torch.Tensor.new_ones, torch.Tensor.new_tensor, torch.Tensor.new_zeros, torch.Tensor.to, torch.Tensor.pin_memory, torch.nn.Module.to, torch.nn.Module.to_empty
    *************************************************************************************************************
    
  warnings.warn(msg, ImportWarning)
/usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/contrib/transfer_to_npu.py:260: RuntimeWarning: torch.jit.script and torch.jit.script_method will be disabled by transfer_to_npu, which currently does not support them, if you need to enable them, please do not use transfer_to_npu.
  warnings.warn(msg, RuntimeWarning)
WARNING ⚠️ no model scale passed. Assuming scale='n'.
Transferred 361/397 items from pretrained weights
New https://pypi.org/project/ultralytics/8.4.19 available 😃 Update with 'pip install -U ultralytics'
Ultralytics 8.3.24 🚀 Python-3.11.13 torch-2.1.0 CUDA:0 (Ascend910_9362, 62740MiB)
                                                 CUDA:1 (Ascend910_9362, 62740MiB)
                                                 CUDA:2 (Ascend910_9362, 62740MiB)
                                                 CUDA:3 (Ascend910_9362, 62740MiB)
                                                 CUDA:4 (Ascend910_9362, 62740MiB)
                                                 CUDA:5 (Ascend910_9362, 62740MiB)
                                                 CUDA:6 (Ascend910_9362, 62740MiB)
                                                 CUDA:7 (Ascend910_9362, 62740MiB)
                                                 CUDA:8 (Ascend910_9362, 62740MiB)
                                                 CUDA:9 (Ascend910_9362, 62740MiB)
                                                 CUDA:10 (Ascend910_9362, 62740MiB)
                                                 CUDA:11 (Ascend910_9362, 62740MiB)
                                                 CUDA:12 (Ascend910_9362, 62740MiB)
                                                 CUDA:13 (Ascend910_9362, 62740MiB)
                                                 CUDA:14 (Ascend910_9362, 62740MiB)
                                                 CUDA:15 (Ascend910_9362, 62740MiB)
engine/trainer: task=obb, mode=train, model=./ultralytics/cfg/models/v8/yolov8-obb.yaml, data=./ultralytics/cfg/datasets/DOTAv2.0.yaml, epochs=1, time=None, patience=100, batch=16, imgsz=12000, save=True, save_period=-1, cache=False, device=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, workers=24, project=None, name=train36, exist_ok=False, pretrained=./yolov8n-obb.pt, optimizer=auto, verbose=True, seed=0, deterministic=False, single_cls=False, rect=False, cos_lr=False, close_mosaic=10, resume=False, amp=True, fraction=1.0, profile=False, freeze=None, multi_scale=False, data_shuffle=True, overlap_mask=True, mask_ratio=4, dropout=0.0, val=True, split=val, save_json=False, save_hybrid=False, conf=None, iou=0.7, max_det=300, half=False, dnn=False, plots=True, source=None, vid_stride=1, stream_buffer=False, visualize=False, augment=False, agnostic_nms=False, classes=None, retina_masks=False, embed=None, show=False, save_frames=False, save_txt=False, save_conf=False, save_crop=False, show_labels=True, show_conf=True, show_boxes=True, line_width=None, format=torchscript, keras=False, optimize=False, int8=False, dynamic=False, simplify=True, opset=None, workspace=4, nms=False, lr0=0.01, lrf=0.01, momentum=0.937, weight_decay=0.0005, warmup_epochs=3.0, warmup_momentum=0.8, warmup_bias_lr=0.1, box=7.5, cls=0.5, dfl=1.5, pose=12.0, kobj=1.0, label_smoothing=0.0, nbs=64, hsv_h=0.015, hsv_s=0.7, hsv_v=0.4, degrees=0.0, translate=0.1, scale=0.5, shear=0.0, perspective=0.0, flipud=0.0, fliplr=0.5, bgr=0.0, mosaic=1.0, mixup=0.0, copy_paste=0.0, copy_paste_mode=flip, auto_augment=randaugment, erasing=0.4, crop_fraction=1.0, cfg=None, tracker=botsort.yaml, save_dir=/data/yolov8-train-datav1/modelzoo-GPL/runs/obb/train36
Overriding model.yaml nc=80 with nc=18
WARNING ⚠️ no model scale passed. Assuming scale='n'.

                   from  n    params  module                                       arguments                     
  0                  -1  1       464  ultralytics.nn.modules.conv.Conv             [3, 16, 3, 2]                 
  1                  -1  1      4672  ultralytics.nn.modules.conv.Conv             [16, 32, 3, 2]                
  2                  -1  1      7360  ultralytics.nn.modules.block.C2f             [32, 32, 1, True]             
  3                  -1  1     18560  ultralytics.nn.modules.conv.Conv             [32, 64, 3, 2]                
  4                  -1  2     49664  ultralytics.nn.modules.block.C2f             [64, 64, 2, True]             
  5                  -1  1     73984  ultralytics.nn.modules.conv.Conv             [64, 128, 3, 2]               
  6                  -1  2    197632  ultralytics.nn.modules.block.C2f             [128, 128, 2, True]           
  7                  -1  1    295424  ultralytics.nn.modules.conv.Conv             [128, 256, 3, 2]              
  8                  -1  1    460288  ultralytics.nn.modules.block.C2f             [256, 256, 1, True]           
  9                  -1  1    164608  ultralytics.nn.modules.block.SPPF            [256, 256, 5]                 
 10                  -1  1         0  torch.nn.modules.upsampling.Upsample         [None, 2, 'nearest']          
 11             [-1, 6]  1         0  ultralytics.nn.modules.conv.Concat           [1]                           
 12                  -1  1    148224  ultralytics.nn.modules.block.C2f             [384, 128, 1]                 
 13                  -1  1         0  torch.nn.modules.upsampling.Upsample         [None, 2, 'nearest']          
 14             [-1, 4]  1         0  ultralytics.nn.modules.conv.Concat           [1]                           
 15                  -1  1     37248  ultralytics.nn.modules.block.C2f             [192, 64, 1]                  
 16                  -1  1     36992  ultralytics.nn.modules.conv.Conv             [64, 64, 3, 2]                
 17            [-1, 12]  1         0  ultralytics.nn.modules.conv.Concat           [1]                           
 18                  -1  1    123648  ultralytics.nn.modules.block.C2f             [192, 128, 1]                 
 19                  -1  1    147712  ultralytics.nn.modules.conv.Conv             [128, 128, 3, 2]              
 20             [-1, 9]  1         0  ultralytics.nn.modules.conv.Concat           [1]                           
 21                  -1  1    493056  ultralytics.nn.modules.block.C2f             [384, 256, 1]                 
 22        [15, 18, 21]  1    826489  ultralytics.nn.modules.head.OBB              [18, 1, [64, 128, 256]]       
/usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/utils/storage.py:41: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
  if self.device.type != 'cpu':
YOLOv8-obb summary: 250 layers, 3,086,025 parameters, 3,086,009 gradients, 8.5 GFLOPs

Transferred 361/397 items from pretrained weights
DDP: debug command /usr/local/python3.11.13/bin/python3 -m torch.distributed.run --nproc_per_node 16 --master_port 60989 /root/.config/Ultralytics/DDP/_temp_17u3ywi1281468267296400.py
/usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/contrib/transfer_to_npu.py:309: ImportWarning: 
    *************************************************************************************************************
    The torch.Tensor.cuda and torch.nn.Module.cuda are replaced with torch.Tensor.npu and torch.nn.Module.npu now..
    The torch.cuda.DoubleTensor is replaced with torch.npu.FloatTensor cause the double type is not supported now..
    The backend in torch.distributed.init_process_group set to hccl now..
    The torch.cuda.* and torch.cuda.amp.* are replaced with torch.npu.* and torch.npu.amp.* now..
    The device parameters have been replaced with npu in the function below:
    torch.logspace, torch.randint, torch.hann_window, torch.rand, torch.full_like, torch.ones_like, torch.rand_like, torch.randperm, torch.arange, torch.frombuffer, torch.normal, torch._empty_per_channel_affine_quantized, torch.empty_strided, torch.empty_like, torch.scalar_tensor, torch.tril_indices, torch.bartlett_window, torch.ones, torch.sparse_coo_tensor, torch.randn, torch.kaiser_window, torch.tensor, torch.triu_indices, torch.as_tensor, torch.zeros, torch.randint_like, torch.full, torch.eye, torch._sparse_csr_tensor_unsafe, torch.empty, torch._sparse_coo_tensor_unsafe, torch.blackman_window, torch.zeros_like, torch.range, torch.sparse_csr_tensor, torch.randn_like, torch.from_file, torch._cudnn_init_dropout_state, torch._empty_affine_quantized, torch.linspace, torch.hamming_window, torch.empty_quantized, torch._pin_memory, torch.load, torch.set_default_device, torch.Tensor.new_empty, torch.Tensor.new_empty_strided, torch.Tensor.new_full, torch.Tensor.new_ones, torch.Tensor.new_tensor, torch.Tensor.new_zeros, torch.Tensor.to, torch.Tensor.pin_memory, torch.nn.Module.to, torch.nn.Module.to_empty
    *************************************************************************************************************
    
  warnings.warn(msg, ImportWarning)
/usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/contrib/transfer_to_npu.py:260: RuntimeWarning: torch.jit.script and torch.jit.script_method will be disabled by transfer_to_npu, which currently does not support them, if you need to enable them, please do not use transfer_to_npu.
  warnings.warn(msg, RuntimeWarning)
Ultralytics 8.3.24 🚀 Python-3.11.13 torch-2.1.0 CUDA:0 (Ascend910_9362, 62740MiB)
                                                 CUDA:1 (Ascend910_9362, 62740MiB)
                                                 CUDA:2 (Ascend910_9362, 62740MiB)
                                                 CUDA:3 (Ascend910_9362, 62740MiB)
                                                 CUDA:4 (Ascend910_9362, 62740MiB)
                                                 CUDA:5 (Ascend910_9362, 62740MiB)
                                                 CUDA:6 (Ascend910_9362, 62740MiB)
                                                 CUDA:7 (Ascend910_9362, 62740MiB)
                                                 CUDA:8 (Ascend910_9362, 62740MiB)
                                                 CUDA:9 (Ascend910_9362, 62740MiB)
                                                 CUDA:10 (Ascend910_9362, 62740MiB)
                                                 CUDA:11 (Ascend910_9362, 62740MiB)
                                                 CUDA:12 (Ascend910_9362, 62740MiB)
                                                 CUDA:13 (Ascend910_9362, 62740MiB)
                                                 CUDA:14 (Ascend910_9362, 62740MiB)
                                                 CUDA:15 (Ascend910_9362, 62740MiB)
Overriding model.yaml nc=80 with nc=18
WARNING ⚠️ no model scale passed. Assuming scale='n'.
Transferred 391/397 items from pretrained weights
Freezing layer 'model.22.dfl.conv.weight'
AMP: running Automatic Mixed Precision (AMP) checks...
train: Scanning /data/yolov8-train-datav1/datasets/DOTAv2/labels/train.cache... 1411 images, 1 backgrounds, 2 corrupt: 100%|██████████| 1411/1411 [00:00<?, ?it/s]
train: WARNING ⚠️ /data/yolov8-train-datav1/datasets/DOTAv2/images/train/P0334.jpg: ignoring corrupt image/label: non-normalized or out of bounds coordinates [     1.0565      1.0583]
train: WARNING ⚠️ /data/yolov8-train-datav1/datasets/DOTAv2/images/train/P1872.jpg: ignoring corrupt image/label: non-normalized or out of bounds coordinates [     2.2618      2.2437      2.1674      2.0553      2.0337      2.0154      1.9993       1.998      1.9395      1.8671      1.8864      1.9115      1.9278      2.0144      1.8735      1.8426       1.822      1.8034      1.7907      1.7744      1.7483       1.723      1.6903      1.8182      1.6506      1.6087
      2.2772      2.2118      1.6596       2.178       2.667      2.7528      2.7749      2.4934      2.4721      2.5177      2.5772        2.64       2.818      2.8019      2.8344      2.8832      2.8976      2.8004       2.751      2.6603      2.6152      2.5574      2.4091      2.3637      2.2918      2.3171
      2.2527       2.097      2.1098      1.6534        1.61      1.5594      1.0347        1.51      1.5529      1.4456      1.4922      1.3925       1.364      1.3362      1.2735       1.213      1.1513      1.2916      1.1644      1.1396      1.3352      1.4364      1.2961      1.1785      1.1532      1.2294
      1.2884      1.3137       1.033      1.0843      1.0808      1.0927]
val: Scanning /data/yolov8-train-datav1/datasets/DOTAv2/labels/val.cache... 458 images, 0 backgrounds, 0 corrupt: 100%|██████████| 458/458 [00:00<?, ?it/s]
WARNING ⚠️ 'rect=True' is incompatible with DataLoader shuffle, setting shuffle=False
/usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/utils/storage.py:41: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
  if self.device.type != 'cpu':
Plotting labels to /data/yolov8-train-datav1/modelzoo-GPL/runs/obb/train36/labels.jpg... 
optimizer: 'optimizer=auto' found, ignoring 'lr0=0.01' and 'momentum=0.937' and determining best 'optimizer', 'lr0' and 'momentum' automatically... 
optimizer: NpuFusedAdam(lr=0.000714, momentum=0.9) with parameter groups 63 weight(decay=0.0), 73 weight(decay=0.0005), 72 bias(decay=0.0)
Image sizes 12000 train, 12000 val
Using 384 dataloader workers
Logging results to /data/yolov8-train-datav1/modelzoo-GPL/runs/obb/train36
Starting training for 1 epochs...

      Epoch    GPU_mem   box_loss   cls_loss   dfl_loss  Instances       Size
  0%|          | 0/89 [00:00<?, ?it/s][E compiler_depend.ts:443] NPU out of memory. NPUWorkspaceAllocator tried to allocate 1.07 GiB(NPU 9; 61.28 GiB total capacity; 790.04 MiB free). If you want to reduce memory usage, take a try to set the environment variable TASK_QUEUE_ENABLE=1.

[ERROR] 2026-03-04-14:33:03 (PID:240505, Device:9, RankID:9) ERR00006 PTA memory error
Exception raised from malloc at build/CMakeFiles/torch_npu.dir/compiler_depend.ts:426 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x68 (0xfffccb37d898 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)
frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::string const&) + 0x6c (0xfffccb3362a8 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)
frame #2: <unknown function> + 0x8dd988 (0xfffcb885d988 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #3: <unknown function> + 0x8de160 (0xfffcb885e160 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #4: <unknown function> + 0x8d80fc (0xfffcb88580fc in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #5: <unknown function> + 0x26ed31c (0xfffcba66d31c in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #6: <unknown function> + 0xe6ac64 (0xfffcb8deac64 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #7: <unknown function> + 0x26998ec (0xfffcba6198ec in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #8: <unknown function> + 0x8bec78 (0xfffcb883ec78 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #9: <unknown function> + 0x8c1688 (0xfffcb8841688 in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #10: <unknown function> + 0x8bd7cc (0xfffcb883d7cc in /usr/local/python3.11.13/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #11: <unknown function> + 0x946ec (0xfffccb3a46ec in /usr/local/python3.11.13/lib/python3.11/site-packages/torch/lib/libc10.so)
frame #12: <unknown function> + 0x80398 (0xfffcd7630398 in /lib/aarch64-linux-gnu/libc.so.6)
frame #13: <unknown function> + 0xe9e9c (0xfffcd7699e9c in /lib/aarch64-linux-gnu/libc.so.6)

[E compiler_depend.ts:443] NPU out of memory. NPUWorkspaceAllocator tried to allocate 1.07 GiB(NPU 11; 61.28 GiB total capacity; 789.24 MiB free). If you want to reduce memory usage, take a try to set the environment variable TASK_QUEUE_ENABLE=1.
RuntimeError    : return forward_call(*args, **kwargs)
The Inner error is reported as above. The process exits for this inner error, and the current working operator name is aclnnConvolution.
Since the operator is called asynchronously, the stacktrace may be inaccurate. If you want to get the accurate stacktrace, please set the environment variable ASCEND_LAUNCH_BLOCKING=1.
Note: ASCEND_LAUNCH_BLOCKING=1 will force ops to run in synchronous mode, resulting in performance degradation. Please unset ASCEND_LAUNCH_BLOCKING in time after debugging.
[ERROR] 2026-03-04-14:38:20 (PID:269279, Device:6, RankID:6) ERR00100 PTA call acl api failed.
[PID: 269279] 2026-03-04-14:38:20.443.950 Memory_Allocation_Failure(EL0004): Failed to allocate memory.
        Possible Cause: Available memory is insufficient.
        Solution: Close applications not in use.
        TraceBack (most recent call last):
        alloc device memory failed, runtime result = 207001[FUNC:ReportCallError][FILE:log_inner.cpp][LINE:162]

    return forward_call(*args, **kwargs)
                    ^ ^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^
  File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/nn/modules/batchnorm.py", line 171, in forward
  File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/nn/modules/batchnorm.py", line 171, in forward
    return F.batch_norm(    
return F.batch_norm(
                     ^ ^^^^^^^^^^^^^^^^^^^^^^^^
^
  File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/nn/functional.py", line 2478, in batch_norm
  File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/nn/functional.py", line 2478, in batch_norm
    return torch.batch_norm(
           return torch.batch_norm( 
     ^ ^ ^ ^ ^ ^ ^ ^ ^ ^^^^^^^^^^^^^^^^
^^^RuntimeError^: ^The Inner error is reported as above. The process exits for this inner error, and the current working operator name is aclnnConvolution.
Since the operator is called asynchronously, the stacktrace may be inaccurate. If you want to get the accurate stacktrace, please set the environment variable ASCEND_LAUNCH_BLOCKING=1.
Note: ASCEND_LAUNCH_BLOCKING=1 will force ops to run in synchronous mode, resulting in performance degradation. Please unset ASCEND_LAUNCH_BLOCKING in time after debugging.
[ERROR] 2026-03-04-14:38:20 (PID:269283, Device:10, RankID:10) ERR00100 PTA call acl api failed.
[PID: 269283] 2026-03-04-14:38:20.440.030 Memory_Allocation_Failure(EL0004): Failed to allocate memory.
        Possible Cause: Available memory is insufficient.
        Solution: Close applications not in use.
        TraceBack (most recent call last):
        alloc device memory failed, runtime result = 207001[FUNC:ReportCallError][FILE:log_inner.cpp][LINE:162]
^
^^^
RuntimeError: The Inner error is reported as above. The process exits for this inner error, and the current working operator name is aclnnConvolution.
Since the operator is called asynchronously, the stacktrace may be inaccurate. If you want to get the accurate stacktrace, please set the environment variable ASCEND_LAUNCH_BLOCKING=1.
Note: ASCEND_LAUNCH_BLOCKING=1 will force ops to run in synchronous mode, resulting in performance degradation. Please unset ASCEND_LAUNCH_BLOCKING in time after debugging.
[ERROR] 2026-03-04-14:38:20 (PID:269285, Device:12, RankID:12) ERR00100 PTA call acl api failed.
[PID: 269285] 2026-03-04-14:38:20.440.466 Memory_Allocation_Failure(EL0004): Failed to allocate memory.
        Possible Cause: Available memory is insufficient.
        Solution: Close applications not in use.
        TraceBack (most recent call last):
        alloc device memory failed, runtime result = 207001[FUNC:ReportCallError][FILE:log_inner.cpp][LINE:162]

/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmp8qg9me0p'>
  _warnings.warn(warn_message, ResourceWarning)
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmpdgsxsl44'>
  _warnings.warn(warn_message, ResourceWarning)
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmp3q3bq6by'>
  _warnings.warn(warn_message, ResourceWarning)
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmp5cjjo_g_'>
  _warnings.warn(warn_message, ResourceWarning)
Exception in thread Thread-1 (_pin_memory_loop):
Traceback (most recent call last):
  File "/usr/local/python3.11.13/lib/python3.11/threading.py", line 1045, in _bootstrap_inner
    self.run()
  File "/usr/local/python3.11.13/lib/python3.11/threading.py", line 982, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/utils/data/_utils/pin_memory.py", line 54, in _pin_memory_loop
    do_one_step()
  File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/utils/data/_utils/pin_memory.py", line 31, in do_one_step
    r = in_queue.get(timeout=MP_STATUS_CHECK_INTERVAL)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/queues.py", line 122, in get
    return _ForkingPickler.loads(res)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/multiprocessing/reductions.py", line 355, in rebuild_storage_fd
    fd = df.detach()
         ^^^^^^^^^^^
  File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/resource_sharer.py", line 57, in detach
    with _resource_sharer.get_connection(self._id) as conn:
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/resource_sharer.py", line 86, in get_connection
    c = Client(address, authkey=process.current_process().authkey)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/connection.py", line 526, in Client
    deliver_challenge(c, authkey)
  File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/connection.py", line 757, in deliver_challenge
    response = connection.recv_bytes(256)        # reject large message
               ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/connection.py", line 216, in recv_bytes
    buf = self._recv_bytes(maxlength)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/connection.py", line 430, in _recv_bytes
    buf = self._recv(4)
          ^^^^^^^^^^^^^
  File "/usr/local/python3.11.13/lib/python3.11/multiprocessing/connection.py", line 395, in _recv
    chunk = read(handle, remaining)
            ^^^^^^^^^^^^^^^^^^^^^^^
ConnectionResetError: [Errno 104] Connection reset by peer
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmpe2kuimr_'>
  _warnings.warn(warn_message, ResourceWarning)
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmpr5nd3zmv'>
  _warnings.warn(warn_message, ResourceWarning)
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmph9wwtu4m'>
  _warnings.warn(warn_message, ResourceWarning)
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmpbm_i107o'>
  _warnings.warn(warn_message, ResourceWarning)
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmp36l33qpk'>
  _warnings.warn(warn_message, ResourceWarning)
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmprnj7k2ee'>
  _warnings.warn(warn_message, ResourceWarning)
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmpngc4frxb'>
  _warnings.warn(warn_message, ResourceWarning)
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmp7gpmavmu'>
  _warnings.warn(warn_message, ResourceWarning)
Exception in thread Thread-1 (_pin_memory_loop):
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmpe4vb6b5d'>
  _warnings.warn(warn_message, ResourceWarning)
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmpg9zkg84j'>
  _warnings.warn(warn_message, ResourceWarning)
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmpbnpkzs4_'>
  _warnings.warn(warn_message, ResourceWarning)
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmpgs1t4dxh'>
  _warnings.warn(warn_message, ResourceWarning)
terminate called without an active exception
[2026-03-04 14:38:30,741] torch.distributed.elastic.multiprocessing.api: [WARNING] Sending process 269273 closing signal SIGTERM
[2026-03-04 14:38:30,741] torch.distributed.elastic.multiprocessing.api: [WARNING] Sending process 269275 closing signal SIGTERM
[2026-03-04 14:38:30,741] torch.distributed.elastic.multiprocessing.api: [WARNING] Sending process 269276 closing signal SIGTERM
[2026-03-04 14:38:30,741] torch.distributed.elastic.multiprocessing.api: [WARNING] Sending process 269280 closing signal SIGTERM
[2026-03-04 14:38:30,741] torch.distributed.elastic.multiprocessing.api: [WARNING] Sending process 269281 closing signal SIGTERM
[2026-03-04 14:38:30,741] torch.distributed.elastic.multiprocessing.api: [WARNING] Sending process 269283 closing signal SIGTERM
[2026-03-04 14:38:30,741] torch.distributed.elastic.multiprocessing.api: [WARNING] Sending process 269285 closing signal SIGTERM
[2026-03-04 14:38:30,741] torch.distributed.elastic.multiprocessing.api: [WARNING] Sending process 269286 closing signal SIGTERM
[2026-03-04 14:38:30,741] torch.distributed.elastic.multiprocessing.api: [WARNING] Sending process 269287 closing signal SIGTERM
[2026-03-04 14:38:36,161] torch.distributed.elastic.multiprocessing.api: [ERROR] failed (exitcode: 1) local_rank: 1 (pid: 269274) of binary: /usr/local/python3.11.13/bin/python3
Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/distributed/run.py", line 810, in <module>
    main()
  File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper
    return f(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^
  File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/distributed/run.py", line 806, in main
    run(args)
  File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/distributed/run.py", line 797, in run
    elastic_launch(
  File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 134, in __call__
    return launch_agent(self._config, self._entrypoint, list(args))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/python3.11.13/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent
    raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
============================================================
/root/.config/Ultralytics/DDP/_temp_diabdmw6281471978158544.py FAILED
------------------------------------------------------------
Failures:
[1]:
  time      : 2026-03-04_14:38:30
  host      : loaclhost
  rank      : 4 (local_rank: 4)
  exitcode  : 1 (pid: 269277)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
[2]:
  time      : 2026-03-04_14:38:30
  host      : loaclhost
  rank      : 5 (local_rank: 5)
  exitcode  : 1 (pid: 269278)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
[3]:
  time      : 2026-03-04_14:38:30
  host      : loaclhost
  rank      : 6 (local_rank: 6)
  exitcode  : 1 (pid: 269279)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
[4]:
  time      : 2026-03-04_14:38:30
  host      : loaclhost
  rank      : 9 (local_rank: 9)
  exitcode  : 1 (pid: 269282)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
[5]:
  time      : 2026-03-04_14:38:30
  host      : loaclhost
  rank      : 11 (local_rank: 11)
  exitcode  : 1 (pid: 269284)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
[6]:
  time      : 2026-03-04_14:38:30
  host      : loaclhost
  rank      : 15 (local_rank: 15)
  exitcode  : 1 (pid: 269288)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
------------------------------------------------------------
Root Cause (first observed failure):
[0]:
  time      : 2026-03-04_14:38:30
  host      : loaclhost
  rank      : 1 (local_rank: 1)
  exitcode  : 1 (pid: 269274)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
============================================================
Traceback (most recent call last):
  File "/data/yolov8-train-datav1/modelzoo-GPL/built-in/PyTorch/Official/cv/object_detection/Yolov8_for_PyTorch/train.py", line 43, in <module>
    model.train(
  File "/data/yolov8-train-datav1/modelzoo-GPL/built-in/PyTorch/Official/cv/object_detection/Yolov8_for_PyTorch/ultralytics/engine/model.py", line 784, in train
    self.trainer.train()
  File "/data/yolov8-train-datav1/modelzoo-GPL/built-in/PyTorch/Official/cv/object_detection/Yolov8_for_PyTorch/ultralytics/engine/trainer.py", line 203, in train
    raise e
  File "/data/yolov8-train-datav1/modelzoo-GPL/built-in/PyTorch/Official/cv/object_detection/Yolov8_for_PyTorch/ultralytics/engine/trainer.py", line 201, in train
    subprocess.run(cmd, check=True)
  File "/usr/local/python3.11.13/lib/python3.11/subprocess.py", line 571, in run
    raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['/usr/local/python3.11.13/bin/python3', '-m', 'torch.distributed.run', '--nproc_per_node', '16', '--master_port', '52031', '/root/.config/Ultralytics/DDP/_temp_diabdmw6281471978158544.py']' returned non-zero exit status 1.
[ERROR] 2026-03-04-14:38:38 (PID:269257, Device:-1, RankID:-1) ERR99999 UNKNOWN application exception
/usr/local/python3.11.13/lib/python3.11/tempfile.py:934: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmpsbdw3l4w'>
  _warnings.warn(warn_message, ResourceWarning)

我要发帖子

我们使用cookie来确保您的高速浏览体验。继续浏览本站，即表示您同意我们使用cookie。查看详情