一、基础环境

8.0.RC2 cann

二、操作步骤

执行hccl测试脚本，出现kernel相关报错，tookit和kernel包均已经安装。

三、问题现象

四、测试脚本

 

#!/bin/bash 

 

source /usr/local/Ascend/ascend-toolkit/set_env.sh 

source /usr/local/Ascend/mindie/set_env.sh 

 

export HCCL_CONNECT_TIMEOUT=300 

export PYTORCH_NPU_ALLOC_CONF="expandable_segments:True" 

 

python test_pytorch.py 

test_pytorch.py内容如下：

import os

import torch

import torch_npu

import torch.distributed as dist

from torch.multiprocessing import Process

def init_process(rank, size, fn, backend='hccl'):

    os.environ['MASTER_ADDR'] = '127.0.0.1'

    os.environ['MASTER_PORT'] = '29500'

    dist.init_process_group(backend=backend, rank=rank, world_size=size)

    fn(rank, size)

    dist.destroy_process_group()

def run_collective_communication(rank, size):

    device = torch.device(f'npu:{rank}')

    tensor = torch.tensor([rank], dtype=torch.float32).to(device)

   

    # All Reduce

    dist.all_reduce(tensor, op=dist.ReduceOp.SUM)

    print(f"Rank {rank} after all_reduce SUM: {tensor.item()}")

   

    # Broadcast

    tensor.fill_(rank)

    dist.broadcast(tensor, src=0)

    print(f"Rank {rank} after broadcast from rank 0: {tensor.item()}")

   

    # All Gather

    gather_tensor = [torch.zeros(1, dtype=torch.float32).to(device) for _ in range(size)]

    dist.all_gather(gather_tensor, tensor)

    if rank == 0:

        print(f"Rank {rank} after all_gather: {[t.item() for t in gather_tensor]}")

def run(rank, size):

    run_collective_communication(rank, size)

if __name__ == "__main__":

    size = 4

    os.environ['HCCL_CONNECT_TIMEOUT'] = '300'

    os.environ['PYTORCH_NPU_ALLOC_CONF'] = "expandable_segments:True"

   

    processes = []

    for rank in range(size):

        p = Process(target=init_process, args=(rank, size, run))

        p.start()

        processes.append(p)

   

    for p in processes:

        p.join()