Querying the Topology Information

Background

To cope with complex network topologies, communication operators need to select the most matched algorithm based on the topology of the communicator. Therefore, the HCCL control plane provides the function of querying the topology information.

Topology Information

The following table describes the topology information that can be queried using the HCCL control plane APIs.

Topology Information	Query API
Obtains the rank ID of a device in a specified communicator.	HcclGetRankId
Queries the number of ranks in a specified communicator.	HcclGetRankSize
Queries the list of topology layers that contain the current rank and the number of topology layers.	HcclRankGraphGetLayers
Returns the list of all rank IDs and the number of ranks in the topology instance where the current rank is located at a specified topology layer in a specified communicator.	HcclRankGraphGetRanksByLayer
Returns the number of ranks in the topology instance where the current rank is located at a specified topology layer in a specified communicator.	HcclRankGraphGetRankSizeByLayer
Returns the topology type of the topology layer where the current rank is located in a specified communicator.	HcclRankGraphGetTopoTypeByLayer
Queries the number of topology instances at a specified topology layer and the number of ranks in each instance in a specified communicator.	HcclRankGraphGetInstSizeListByLayer
Queries information about the communication connection between the source rank and the destination rank at a specified topology layer in a specified communicator.	HcclRankGraphGetLinks

Query the ID of the current rank in a communicator.

u32 userRank = INVALID_VALUE_RANKID;
HcclResult ret = HcclGetRankId(comm, &userRank);
if (userRank == root && sendBuf == nullptr) {     // send_buff of the root rank cannot be empty.
    return HCCL_E_PTR;
}

Query the number of ranks in a communicator.

u32 rankSize = INVALID_VALUE_RANKSIZE;
HcclResult ret = HcclGetRankSize(comm, &rankSize);
if (userRank >= rankSize) {     // rank_id is out of range.
    return HCCL_E_PARA;
}

Query information about the link between the current device and device 0 on the server.

u32 dstRank = 0;
u32 srcRank = rank;
CommLink *linkList = nullptr;
u32 listSize = 0;
HcclResult ret = HcclRankGraphGetLinks(comm, 0, srcRank, dstRank, &linkList, &listSize);
for (u32 i = 0; i < listSize; ++i) {
    CommLink& currentLink = linkList[i]; // Enumerate all link objects.
    if (currentLink.linkAttr.linkProtocol == CommProtocol::COMM_PROTOCOL_HCCS) {
        // Determine the HCCS link.
    }
}

Query the number of SuperPoDs in the communicator where the device is located.

u32 *netlayers = nullptr;
u32 netLayersNum = 0;
HcclResult ret = HcclRankGraphGetLayers(comm, &netlayers, &netLayersNum); // Obtain the communication network layers in the communicator.
u32 superPodNum;
u32 level1RankListNum = 0;
u32 *level1SizeList = nullptr;
if (netLayersNum == 3) { // SuperPoD scenario
    ret = HcclRankGraphGetInstSizeListByLayer(comm, 1, &level1SizeList, &level1RankListNum);
    superPodNum = level1RankListNum;
}

Sample Code

// The mesh topology of four devices on a one server is used as an example.
struct TopoInfo {
    uint32_t rankId; // Uniquely identifies a rank.
    uint32_t rankSize; // Number of ranks participating in collective communication.
    std::vector<u32> rankList; // A collection of rank IDs involved in collective communication.
    CommTopo topoType; // Link type, such as COMM_TOPO_1DMESH or COMM_TOPO_CLOS.
    std::vector<HcclCHannelDesc> channels; // Links between the current rank and other cards.
};
HcclResult FillSimpleTopoInfo(HcclComm comm, TopoInfo &topoInfo){
    HcclResult ret = HcclGetRankId(comm, &topoInfo.rankId);
    // Verify the result.
    ret = HcclGetRankSize(comm, &topoInfo.rankSize);
    // Verify the result.
    uint32_t *ranks = nullptr;
    uint32_t rankNum = 0;
    // Because there are four devices on one server, netLayer is 0. The ranks obtained under netLayer 0 are the collection of rank IDs involved in collective communication.
    ret = HcclRankGraphGetRanksByLayer(comm, 0, &ranks, &rankNum);
    // Verify the result.
    for (size_t index = 0; index < rankNum; index++) {
        topoInfo.rankList.push_back(ranks[index]);
    }
    uint32_t topoInstNum = 0;
    uint32_t *topoInsts;
    ret = HcclRankGraphGetTopoInstsByLayer(comm, 0, &topoInsts, &topoInstNum);
    // Verify the result.
    // Because there are four devices on one server, only one topoInst is available. Therefore, topoInsts = [0] and topoInstNum = 1.
    // Obtain the physical link type of the device through topoInst.
    ret = HcclRankGraphGetTopoTypeByLayer(comm, 0, topoInsts[0], &topoInfo.topoType);
    // Verify the result.
    // Calculate the required channels.
    for (auto remoteRankId : topoInfo.rankList) {
       if (remoteRankId == topoInfo.rankId) {continue;}
       CommLink *linkList = nullptr;
       u32 listSize = 0;
       ret = HcclRankGraphGetLinks(comm, netLayer, myRank, remoteRankId, &linkList, &listSize);
       for (uint32_t idx = 0; idx < listSize; idx++) {
           HcclChannelDesc channelDesc;
           HcclChannelDescInit(&channelDesc, 1);
           channelDesc.remoteRank = remoteRankId;
           CommLink link = linkList[idx];
           channelDesc.localEndpoint.protocol = link.srcEndpointDesc.protocol;
           channelDesc.localEndpoint.commAddr = link.srcEndpointDesc.commAddr;
           channelDesc.localEndpoint.loc = link.srcEndpointDesc.loc;
           channelDesc.remoteEndpoint.protocol = link.dstEndpointDesc.protocol;
           channelDesc.remoteEndpoint.commAddr = link.dstEndpointDesc.commAddr;
           channelDesc.remoteEndpoint.loc = link.dstEndpointDesc.loc;
           channelDesc.channelProtocol = link.linkAttr.linkProtocol;
           channelDesc.notifyNum = 3;
           topoInfo.channels.push_back(channelDesc);
       }
    }
    return ret;    
}