启动方式：python -u -m torch.distributed.launch --nproc_per_node 2 tool/train.py --config data/config/pascal/pascal_asgnet_split0_resnet50.yaml

代码： 

import os  

import random 

import time 

import cv2 

import numpy as np 

import logging 

import argparse 

import torch_npu 

#自动映射cuda API到npu的代码 

from torch_npu.contrib import transfer_to_npu 

#os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 

#os.environ["CUDA_VISIBLE_DEVICES"] = '0'  #只让设备0可见 

from torch.utils.data import DataLoader, DistributedSampler 

import torch 

import torch.nn as nn 

import torch.nn.functional as F 

import torch.nn.parallel 

import torch.utils.data 

import torch.multiprocessing as mp 

import torch.distributed as dist 

 

import sys 

sys.path.append('/home/ma-user/work/ASGNet-main') 

from model import * 

from util import dataset 

from util import transform, config 

from util.util import AverageMeter, poly_learning_rate, intersectionAndUnionGPU 

 

cv2.ocl.setUseOpenCL(False) 

cv2.setNumThreads(0) 

 

def get_parser(): 

    parser = argparse.ArgumentParser(description='PyTorch Semantic Segmentation') 

    parser.add_argument('--config', type=str, default='config/ade20k/ade20k_pspnet50.yaml', help='config file') 

    parser.add_argument("--local_rank", type=int, required=True, help='local rank for DistributedDataParallel') 

    #parser.add_argument("--local-rank", "--local_rank", type=int) 

    parser.add_argument('opts', help='see config/ade20k/ade20k_pspnet50.yaml for all options', default=None, nargs=argparse.REMAINDER) 

    args = parser.parse_args() 

    assert args.config is not None 

    cfg = config.load_cfg_from_cfg_file(args.config) 

    if args.opts is not None: 

        cfg = config.merge_cfg_from_list(cfg, args.opts) 

    cfg.LOCAL_RANK = args.local_rank 

    return cfg 

 

 

def main(): 

    args = get_parser() 

    if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ: 

        rank = int(os.environ["RANK"]) 

        world_size = int(os.environ['WORLD_SIZE']) 

        print(f"RANK and WORLD_SIZE in environ: {rank}/{world_size}") 

    else: 

        rank = -1 

        world_size = -1 

   # args.LOCAL_RANK=int(os.environ["LOCAL_RANK"])  

    torch_npu.npu.set_device(args.LOCAL_RANK)  #换个方法设置device  like npu:0 

    torch.distributed.init_process_group(backend='hccl', world_size=world_size, rank=rank) 

     

     

    model = eval(args.arch).Model(args) 

    model=model.npu() 

 

 

if __name__ == '__main__': 

    main() 

报错：RuntimeError: 0INTERNAL ASSERT FAILED at "/usr1/workspace/FPTA_Daily_v1.11.0_py37/CODE/torch_npu/csrc/core/npu/NPUStream.cpp":145, please report a bug to PyTorch. Could not compute stream ID for 0xffff74b56048 on device -1 (something has gone horribly wrong!) 

terminate called after throwing an instance of 'c10::Error' 

  what():  0INTERNAL ASSERT FAILED at "/usr1/workspace/FPTA_Daily_v1.11.0_py37/CODE/torch_npu/csrc/core/npu/NPUStream.cpp":145, please report a bug to PyTorch. Could not compute stream ID for 0xffff74b56048 on device -1 (something has gone horribly wrong!)