Documentation
Download

Samples

This section provides a complete code example for implementing the RPing function.

The RPing function on a single server (with 8 devices) is implemented.

Sample Code (Atlas 350 Accelerator Card)

The sample code includes the rping_test.cc and rping_test.h files.

  • The header file rping_test.h defines the IP address of the device on a single server (with 8 devices). The details are as follows:
     1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
    #include <stdio.h>
#include <stdint.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include "acl/acl.h"
#include "acl/acl_prof.h"
#include "hccl/hccn_rping.h"
// On a single server (with 8 devices), the following EIDs are for reference only.
#define ipLen_ 33
char device_eid[8][8][33] = {
        {"", "000000000008030000100000df120900", "000000000000030000100000df120100", "000000000007030000100000df120800", "000000000006030000100000df120700", "000000000004030000100000df120500", "000000000005030000100000df120600", "000000000003030000100000df120400"},
    // src=1
    {"000000000007030000100000df122801", "", "", "", "", "", "", ""},
    // src=2
    {"000000000000030000100000df124102", "", "", "", "", "", "", ""},
    // src=3
    {"000000000007030000100000df126803", "", "", "", "", "", "", ""},
    // src=4
    {"000000000047030000100000df129804", "", "", "", "", "", "", ""},
    // src=5
    {"000000000047030000100000df12b805", "", "", "", "", "", "", ""},
    // src=6
    {"000000000047030000100000df12d806", "", "", "", "", "", "", ""},
    // src=7
    {"000000000047030000100000df12f807", "", "", "", "", "", "", ""},
};
  • The rping_test.cc file implements the RPing function. The sample code is as follows:
      1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
    #include <chrono>
#include <vector>
#include <thread>
#include <atomic>
#include "rping_test.h"
int singleDevAllProc(int devId, std::vector<int> devices, int devClientId, std::atomic<bool> *isStop)
{
    // Determine the number of sent packets and the number of targets in advance for dynamic buffer calculation.
    const uint32_t pktNum = 10;
    int targetNum = 0;
    for (size_t i = 0; i < devices.size(); i++) {
        if (devices[i] != devClientId) {
            targetNum++;
        }
    }
    // Initialize the device.
    HccnRpingInitAttr *initAttr = new HccnRpingInitAttr();
    initAttr->mode = HCCN_RPING_MODE_UB; // Network type. Currently, A5 supports the UB type.
    initAttr->port = 13886; // Listening port, which can be an unused port.
    initAttr->npuNum = 128; // Total number of NPUs involved in the detection.
    size_t needSize = (size_t)pktNum * 2048 * targetNum;
    initAttr->bufferSize = ((needSize + 4095) / 4096 + 1) * 4096; // 4 KB alignment + margin
    initAttr->ipAddr = new char[ipLen_];
    // Use the port EID of the device facing the client. The client uses the port facing the first target as a placeholder.
    int peerId = devClientId;
    for (size_t i = 0; i < devices.size(); i++) {
        if (devices[i] != devId) {
            peerId = devices[i];
            break;
        }
    }
    strcpy(initAttr->eid, device_eid[devId][peerId]);
    HccnRpingCtx rpingCtx = nullptr;
    aclrtSetDevice(devId); // Specify a device before initialization.
    HccnResult ret = HccnRpingInit(devId, initAttr, &rpingCtx);
    if (ret != HCCN_SUCCESS) {
        printf("device init failed.\n");
        return -1;
    }
    printf("rpingCtx [%p]", rpingCtx);
    printf("device[%d] init success!\n", devId);
    if (devId != devClientId) {
        sleep(20); // After the target device is started, it needs to keep waiting for the ping request.
        while (isStop->load() == false) {
            sleep(1);
            continue;
        }
        HccnRpingDeinit(rpingCtx);
        delete[] initAttr->ipAddr;
        delete initAttr;
        return 0;
    }
    // Add the target (skip the client itself).
    std::vector<int> targets;
    for (size_t i = 0; i < devices.size(); i++) {
        if (devices[i] != devClientId) {
            targets.push_back(devices[i]);
        }
    }
    HccnRpingTargetInfo *target = new HccnRpingTargetInfo[targetNum]();
    for (int i = 0; i < targetNum; i++) {
        int devTargetId = targets[i];
        target[i].srcPort = 0; // TCP/UDP source port.
        target[i].sl = 0; // Service level of the RDMA NIC. Value range: [0, 7].
        target[i].tc = 0; // Traffic class of the RDMA NIC. Set it to DSCP value of RoCE packets × 4.
        target[i].port = 13886; // Socket connection port number, which is the same as the port number of the target device during initialization.
        target[i].payloadLen = 12;
        char payload[12] = "hellotarget";
        target[i].srcIp = nullptr;
        target[i].dstIp = nullptr;
        strcpy(target[i].payload, payload);
        target[i].addrType = 1;
        target[i].srcEid = new char[ipLen_];
        target[i].dstEid = new char[ipLen_];
        strcpy (target[i].srcEid, device_eid[devClientId][devTargetId]); // Client-side port
        strcpy (target[i].dstEid, device_eid[devTargetId][devClientId]); // Target-side port
    }
    ret = HccnRpingAddTarget(rpingCtx, targetNum, target);
    if (ret != HCCN_SUCCESS) {
        for (int i = 0; i < targetNum; i++) {
            printf("target[%d] srcEid:%s dstEid:%s addrType:%d port:%d payloadLen:%d\n", i, target[i].srcEid, target[i].dstEid, target[i].addrType, target[i].port, target[i].payloadLen);
        }
        delete[] target;
        printf("device add target failed. ret = %d\n", (int)ret);
        return -1;
    }
    printf("device[%d] ping targets:\n", devId);
    for (int i =0; i < targetNum; i++)
    {
        printf("[eid:%s->%s]\n", target[i].srcEid, target[i].dstEid);
    }
    printf("device[%d] add target success!\n", devId);
    // Start the ping request.
    uint32_t interval = 1;  // ms
    uint32_t timeout = 100; // ms
    ret = HccnRpingBatchPingStart(rpingCtx, pktNum, interval, timeout);
    if (ret != HCCN_SUCCESS) {
        delete[] target;
        printf("device start ping failed.\n");
        return -1;
    }
    printf("device[%d] start ping!\n", devId);
    // Obtain the result.
    HccnRpingResultInfo *result = new HccnRpingResultInfo[targetNum];
    HccnResult hccnRet = HCCN_E_AGAIN;
    while(hccnRet == HCCN_E_AGAIN) {
        sleep(1);
        hccnRet =  HccnRpingGetResult(rpingCtx, targetNum, target, result);
    }
    if (hccnRet != HCCN_SUCCESS) {
        delete[] target;
        delete[] result;
        printf("device get result failed.\n");
        return -1;
    }
    for (int i = 0; i < targetNum; i++) {
        printf("target[device %d] txPkt[%u] rxPkt[%u] minRTT[%u] maxRTT[%u] avgRTT[%u] state[%u]\n",
            targets[i],
            result[i].txPkt,
            result[i].rxPkt,
            result[i].minRTT,
            result[i].maxRTT,
            result[i].avgRTT,
            result[i].state);
    }
    unsigned int payloadLenOutput = 0;
    void* payloadOutput = nullptr;
    HccnRpingGetPayload(rpingCtx, &payloadOutput, &payloadLenOutput);
    int payloadNum = 0;
    for (int i = 0; i < targetNum; i++) {
        payloadNum += result[i].rxPkt;
    }
    if (payloadOutput == nullptr || payloadLenOutput == 0) {
        printf("payload is empty,skip printing.\n");
    }
    else {
        for (int i = 0; i < payloadNum; i++) {
                HccnRpingPayloadHead *head = static_cast<HccnRpingPayloadHead*>(payloadOutput);
                printf("[%dth] srcIp:%s, dstIp:%s, payloadLen:%d, t1:%llu %llu, t2:%llu %llu, t3:%llu %llu, t4:%llu %llu, task id:%u\n",
                        i,
                                head->srcIp, head->dstIp,
                                head->payloadLen,
                                head->t1.sec, head->t1.usec,
                                head->t2.sec, head->t2.usec,
                                head->t3.sec, head->t3.usec,
                                head->t4.sec, head->t4.usec,
                                head->rpingBatchId);
                char* ptrTmp = static_cast<char*>(payloadOutput);
                ptrTmp += 2048;
                payloadOutput = ptrTmp;
        }
    }
    ret = HccnRpingBatchPingStop(rpingCtx);
    if (ret != HCCN_SUCCESS) {
        delete[] target;
        delete[] result;
        printf("device stop ping failed.\n");
        return -1;
    }
    printf("device[%d] stop ping!\n", devId);

    HccnRpingRemoveTarget(rpingCtx, targetNum, target);
    if (ret != HCCN_SUCCESS) {
        delete[] target;
        delete[] result;
        printf("device remove target failed.\n");
        return -1;
    }
    printf("device[%d] remove target success!\n", devId);
    // Deinitialize memory.
    ret = HccnRpingDeinit(rpingCtx);
    if (ret != HCCN_SUCCESS) {
        printf("device deinit failed.\n");
        return -1;
    }
    printf("device[%d] deinit success!\n", devId);
    for (int i = 0; i < targetNum; i++) {
        delete[] target[i].srcEid;
        delete[] target[i].dstEid;
    }
    delete[] result;
    delete[] target;
    printf("rpingCtx test success!!!\n");
    return 0;
}

int main(int argc, char *argv[])
{
    // Obtain the number of devices.
    int deviceNum = argc - 2;
    // Obtain the number of loops.
    int loop = atoi(argv[1]);
    // Record the client ID.
    int devClientId = atoi(argv[2]);
    // Record all device IDs.
    std::vector<int> devices;
    for (int i = 0; i < deviceNum; i++) {
        int dev = atoi(argv[i + 2]);
        bool isRepeat = false;
        for (int j = 0; j < devices.size(); j++) {
            if (dev == devices[j]) {
                isRepeat = true;
                printf("%d is repeat!\n", dev);
                break;
            }
        }
        if (!isRepeat) {
            printf("dev: %d, isrepeat: %d\n", dev, isRepeat);
           devices.push_back(dev);
        }
    }
    std::vector<std::thread> test_threads;
    std::atomic<bool> isStop{false};
    for (int i = 0; i < loop; i++) {
        printf("\n*************divider**************\n\n");
        printf("%dth process start!!\n", i+1);
        for (int j = 0; j < devices.size(); j++) {
            test_threads.push_back(std::thread(singleDevAllProc, devices[j], devices, devClientId, &isStop));
            printf("device[%d] start running!!\n", devices[j]);
        }
        for (int j = 0; j < deviceNum; j++) {
            isStop.store(true);
            if (test_threads[j].joinable()) {
                test_threads[j].join();
                printf("device[%d] stop running!!\n", devices[j]);
            }
        }
        test_threads.clear();
    }
    return 0;
}

Sample Code (Atlas A3 training product/Atlas A3 inference product/Atlas A2 training product/Atlas A2 inference product)

The sample code includes the rping_test.cc and rping_test.h files.

  • The header file rping_test.h defines the IP address of the device on a single server (with 8 devices). The details are as follows:
     1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
    #include <stdio.h>
#include <stdint.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include "acl/acl.h"
#include "acl/acl_prof.h"
#include "hccl/hccn_rping.h"

// On a single server (with 8 devices), the following IP addresses are for reference only.
#define ipLen 16

char deviceIp[8][ipLen] =
{
    "192.168.99.127",
    "192.168.99.128",
    "192.168.99.129",
    "192.168.99.130",
    "192.168.99.131",
    "192.168.99.132",
    "192.168.99.133",
    "192.168.99.134"
};
  • The rping_test.cc file implements the RPing function. The sample code is as follows:
      1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
    #include <chrono>
#include <vector>
#include <thread>
#include <atomic>
#include "rping_test.h"

int singleDevAllProc(int devId, std::vector<int> devices, int devClientId, std::atomic<bool> *isStop)
{
    // Determine the number of sent packets and the number of targets in advance for dynamic buffer calculation.
    const uint32_t pktNum = 10;
    int targetNum = 0;
    for (size_t i = 0; i < devices.size(); i++) {
        if (devices[i] != devClientId) {
            targetNum++;
        }
    }
    // Initialize the device.
    HccnRpingInitAttr *initAttr = new HccnRpingInitAttr();
    initAttr->mode = HCCN_RPING_MODE_ROCE;
    initAttr->port = 13886;
    initAttr->npuNum = 128;
    size_t needSize = (size_t)pktNum * 2048 * targetNum;
    initAttr->bufferSize = ((needSize + 4095) / 4096 + 1) * 4096;
    initAttr->ipAddr = new char[ipLen];
    strcpy(initAttr->ipAddr, deviceIp[devId]);
    HccnRpingCtx rpingCtx = nullptr;
    aclrtSetDevice(devId);
    HccnResult ret = HccnRpingInit(devId, initAttr, &rpingCtx);
    if (ret != HCCN_SUCCESS) {
        printf("device init failed.\n");
        return -1;
    }
    printf("rpingCtx [%p]", rpingCtx);
    printf("device[%d] init success!\n", devId);
    if (devId != devClientId) {
        sleep(20);
        while (isStop->load() == false) {
            sleep(1);
            continue;
        }
        HccnRpingDeinit(rpingCtx);
        delete[] initAttr->ipAddr;
        delete initAttr;
        return 0;
    }
    // Add the target device.
    std::vector<int> targets;
    for (size_t i = 0; i < devices.size(); i++) {
        if (devices[i] != devClientId) {
            targets.push_back(devices[i]);
        }
    }
    HccnRpingTargetInfo *target = new HccnRpingTargetInfo[targetNum];
    for (int i = 0; i < targetNum; i++) {
        int devTargetId = targets[i];
        target[i].srcPort = 0;
        target[i].sl = 4;
        target[i].tc = (33 & 0x3f) << 2;
        target[i].port = 13886;
        target[i].payloadLen = 12;
        target[i].srcIp = new char[ipLen];
        target[i].dstIp = new char[ipLen];
        char payload[12] = "hellotarget";
        strcpy(target[i].payload, payload);
        strcpy(target[i].srcIp, deviceIp[devClientId]);
        strcpy(target[i].dstIp, deviceIp[devTargetId]);
    }
    ret = HccnRpingAddTarget(rpingCtx, targetNum, target);
    if (ret != HCCN_SUCCESS) {
        delete[] target;
        printf("device add target failed.\n");
        return -1;
    }
    printf("device[%d] add target success!\n", devId);
    // Start the ping request.
    uint32_t interval = 1;  // ms
    uint32_t timeout = 100; // ms
    ret = HccnRpingBatchPingStart(rpingCtx, pktNum, interval, timeout);
    if (ret != HCCN_SUCCESS) {
        delete[] target;
        printf("device start ping failed.\n");
        return -1;
    }
    printf("device[%d] start ping!\n", devId);
    // Obtain the result.
    HccnRpingResultInfo *result = new HccnRpingResultInfo[targetNum];
    HccnResult hccnRet = HCCN_E_AGAIN;
    while(hccnRet == HCCN_E_AGAIN) {
        sleep(1);
        hccnRet =  HccnRpingGetResult(rpingCtx, targetNum, target, result);
    }
    if (hccnRet != HCCN_SUCCESS) {
        delete[] target;
        delete[] result;
        printf("device get result failed.\n");
        return -1;
    }
    for (int i = 0; i < targetNum; i++) {
        printf("txPkt[%u] rxPkt[%u] minRTT[%u] maxRTT[%u] avgRTT[%u] state[%u]\n",
            result[i].txPkt,
            result[i].rxPkt,
            result[i].minRTT,
            result[i].maxRTT,
            result[i].avgRTT,
            result[i].state);
    }
    unsigned int payloadLenOutput = 0;
    void* payloadOutput = nullptr;
    HccnRpingGetPayload(rpingCtx, &payloadOutput, &payloadLenOutput);
    int payloadNum = 0;
    for (int i = 0; i < targetNum; i++) {
        payloadNum += result[i].rxPkt;
    }
    for (int i = 0; i < payloadNum; i++) {
        HccnRpingPayloadHead *head = static_cast<HccnRpingPayloadHead*>(payloadOutput);
        printf("[%dth] srcIp:%s, dstIp:%s, payloadLen:%d, t1:%llu %llu, t2:%llu %llu, t3:%llu %llu, t4:%llu %llu, task id:%u\n",
                        i,
                        head->srcIp, head->dstIp,
                        head->payloadLen,
                        head->t1.sec, head->t1.usec,
                        head->t2.sec, head->t2.usec,
                        head->t3.sec, head->t3.usec,
                        head->t4.sec, head->t4.usec,
                        head->rpingBatchId);
        char* ptrTmp = static_cast<char*>(payloadOutput);
        ptrTmp += 2048;
        payloadOutput = ptrTmp;
    }
    ret = HccnRpingBatchPingStop(rpingCtx);
    if (ret != HCCN_SUCCESS) {
        delete[] target;
        delete[] result;
        printf("device stop ping failed.\n");
        return -1;
    }
    printf("device[%d] stop ping!\n", devId);
    HccnRpingRemoveTarget(rpingCtx, targetNum, target);
    if (ret != HCCN_SUCCESS) {
        delete[] target;
        delete[] result;
        printf("device remove target failed.\n");
        return -1;
    }
    printf("device[%d] remove target success!\n", devId);
    // Deinitialize memory.
    ret = HccnRpingDeinit(rpingCtx);
    if (ret != HCCN_SUCCESS) {
        printf("device deinit failed.\n");
        return -1;
    }
    printf("device[%d] deinit success!\n", devId);
    for (int i = 0; i < targetNum; i++) {
        delete[] target[i].srcIp;
        delete[] target[i].dstIp;
    }
    delete[] result;
    delete[] target;
    printf("rpingCtx test success!!!\n");
    return 0;
}

int main(int argc, char *argv[])
{
    // Obtain the number of devices.
    int deviceNum = argc - 2;
    // Obtain the number of loops.
    int loop = atoi(argv[1]);
    // Record the client ID.
    int devClientId = atoi(argv[2]);
    // Record all device IDs.
    std::vector<int> devices;
    for (int i = 0; i < deviceNum; i++) {
        int dev = atoi(argv[i + 2]);
        bool isRepeat = false;
        for (int j = 0; j < devices.size(); j++) {
            if (dev == devices[j]) {
                isRepeat = true;
                printf("%d is repeat!\n", dev);
                break;
            }
        }
        if (!isRepeat) {
            printf("dev: %d, isrepeat: %d\n", dev, isRepeat);
           devices.push_back(dev);
        }
    }
    std::vector<std::thread> test_threads;
    std::atomic<bool> isStop{false};
    for (int i = 0; i < loop; i++) {
        printf("\n*************divider**************\n\n");
        printf("%dth process start!!\n", i+1);
        for (int j = 0; j < devices.size(); j++) {
            test_threads.push_back(std::thread(singleDevAllProc, devices[j], devices, devClientId, &isStop));
            printf("device[%d] start running!!\n", devices[j]);
        }
        for (int j = 0; j < deviceNum; j++) {
            isStop.store(true);
            if (test_threads[j].joinable()) {
                test_threads[j].join();
                printf("device[%d] stop running!!\n", devices[j]);
            }
        }
        test_threads.clear();
    }
    return 0;
}

Makefile Sample

The Makefile required for compilation is as follows:

#
#loading path
#--------------------------------------------------------------------------------------------------------------------------------------------------
CXXFLAGS := -std=c++11\
        -Werror\
        -fstack-protector-strong\
        -fPIE -pie\
        -O2\
        -g\
        -s\
        -Wl,-z,relro\
        -Wl,-z,now\
        -Wl,-z,noexecstack\
        -Wl,--copy-dt-needed-entries
HCCL_INC_DIR = ${ASCEND_DIR}/include
HCCL_LIB_DIR = ${ASCEND_DIR}/lib64
ACL_INC_DIR = ${ASCEND_DIR}/include
ACL_LIB_DIR = ${ASCEND_DIR}/lib64
LIST = rping_test
#
#library flags
#--------------------------------------------------------------------------------------------------------------------------------------------------
LIBS = -L$(HCCL_LIB_DIR) -lhccl_plf\
        -L$(ACL_LIB_DIR) -lascendcl
INCLUDEDIRS = -I$(HCCL_INC_DIR)\
                -I$(ACL_INC_DIR)
#
#make
#--------------------------------------------------------------------------------------------------------------------------------------------------
all:
	@mkdir -p bin
	g++ $(CXXFLAGS) rping_test.cc $(INCLUDEDIRS) -o rping_test $(LIBS)
	@printf "\nRPing_test compile completed\n" 
	mv $(LIST) ./bin
.PHONY: clean
clean:
	rm -rf ./bin/*_test

Dependent Environment Variables

Before compiling the sample code in this section, configure the following environment variables:

source /usr/local/Ascend/cann/set_env.sh
export ASCEND_DIR=/usr/local/Ascend/cann

/usr/local/Ascend indicates the default installation path for CANN software when installed by the root user. If the CANN software is installed by a regular user or to a specified path, replace it with the actual path.

Build Sample

  1. Run the make command to generate the rping_test executable file in the bin directory.
    make
  2. Run the following command to run the rping_test executable file.
    ./bin/rping_test <Number of cycles> <client NPU devlogicId> <target NPU devlogicId>
    • <Number of cycles>: number of cycles from the initialization of the RPing function to the deinitialization of RPing resources.
    • <client NPU devlogicId>: logical ID of the device on the client NPU.
    • <target NPU devlogicId>: logical ID of the target NPU. If there are multiple target NPUs, separate their logical IDs with spaces.

    Command example:

    ./bin/rping_test 1 0 1 2 3 4 5 6 7

    The command indicates that the RPing function is executed once, the client NPU is device 0, and the target NPU consists of 7 NPUs from Device 1 to Device 7.

This site uses cookies to improve your browsing experience. By continuing to browse this site, you agree to our use of cookies.View Details