Asynchronous Model Inference

#include "acl/acl.h"
......

// Obtain the run mode of the AI software stack. The API calls for memory allocation and memory copy are different in different run modes.
aclrtRunMode runMode;
extern bool g_isDevice;
aclrtGetRunMode(&runMode);
g_isDevice = (runMode == ACL_DEVICE);

// 1. Apply for model inference resources.

// The two dots (..) indicate a path relative to the directory of the executable file.
// For example, if the executable file is stored in the out directory, the two dots (..) point to the parent directory of the out directory.
const char* omModelPath = "../model/resnet50.om";

// 1.1 Load the model.
// Obtain the weight memory size and workspace size required for model execution, and allocate memory as required.
aclmdlQuerySize(omModelPath, &modelMemSize_, &modelWeightSize_);
aclrtMalloc(&modelMemPtr_, modelMemSize_, ACL_MEM_MALLOC_HUGE_FIRST);
aclrtMalloc(&modelWeightPtr_, modelWeightSize_, ACL_MEM_MALLOC_HUGE_FIRST);

// Load the offline model file. After the model is successfully loaded, the model ID is returned.
aclmdlLoadFromFileWithMem(modelPath, &modelId_, modelMemPtr_,
        modelMemSize_, modelWeightPtr_, modelWeightSize_);

// 1.2 Obtain the model description based on the model ID.
modelDesc_ = aclmdlCreateDesc();
aclmdlGetDesc(modelDesc_, modelId_);

// 1.3 Use the custom function InitMemPool to initialize the memory pool and store the input and output data for model inference.
// -----Key implementation of the InitMemPool function.-----
string testFile[] = {
        "../data/dog1_1024_683.bin",
        "../data/dog2_1024_683.bin"
    };
size_t fileNum = sizeof(testFile) / sizeof(testFile[0]);
// g_memoryPoolSize indicates the number of memory blocks in the memory pool. The default value is 100.
for (size_t i = 0; i < g_memoryPoolSize; ++i) {
        size_t index = i % (sizeof(testFile) / sizeof(testFile[0]));
        // model process
        uint32_t devBufferSize;
        // Customize the GetDeviceBufferOfFile function to implement the following functions:
        // Obtain the buffer for storing the input image data together with the buffer size, and transfer the image data to the device.
        void *picDevBuffer = Utils::GetDeviceBufferOfFile(testFile[index], devBufferSize);
        aclmdlDataset *input = nullptr;
// Customize the CreateInput function to create data input of type aclmdlDataset for storing the input data of model inference.
        Result ret = CreateInput(picDevBuffer, devBufferSize, input);
        aclmdlDataset *output = nullptr;
        // Customize the CreateOutput function to create data output of type aclmdlDataset for storing the output data of model inference. modelDesc indicates the model description.
        CreateOutput(output, modelDesc);
        {
            std::lock_guard<std::recursive_mutex> lk(freePoolMutex_);
            freeMemoryPool_[input] = output;
        }
}
// -----Key implementation of the InitMemPool function.-----

// 2 Model inference
// 2.1 Create a thread tid and specify the thread with the tid as the thread for processing the callback function in the stream.
// ProcessCallback is a thread function. Call aclrtProcessReport in the thread function, and the callback function is triggered after a specified period of time.
pthread_t tid;
(void)pthread_create(&tid, nullptr, ProcessCallback, &s_isExit);

// 2.2 Specify the thread for processing the callback function in the stream.
aclrtSubscribeReport(tid, stream_);

// 2.3 Create a callback function for processing the model inference result. The callback function is user-defined.
void ModelProcess::CallBackFunc(void *arg)
{
    std::map<aclmdlDataset *, aclmdlDataset *> *dataMap =
        (std::map<aclmdlDataset *, aclmdlDataset *> *)arg;

    aclmdlDataset *input = nullptr;
    aclmdlDataset *output = nullptr;
    MemoryPool *memPool = MemoryPool::Instance();

    for (auto& data : *dataMap) {
        ModelProcess::OutputModelResult(data.second);
        memPool->FreeMemory(data.first, data.second);
    }

    delete dataMap;
}

// 2.4 Customize the ExecuteAsync function to perform model inference.
//-----Start of the key implementation of the user-defined ExecuteAsync function.-----
// g_callbackInterval indicates the callback interval. The default value is 1, indicating that a callback task is delivered every asynchronous inference.
bool isCallback = (g_callbackInterval != 0);
size_t callbackCnt = 0;
std::map<aclmdlDataset *, aclmdlDataset *> *dataMap = nullptr;
aclmdlDataset *input = nullptr;
aclmdlDataset *output = nullptr;
MemoryPool *memPool = MemoryPool::Instance();
    // g_executeTimes indicates the number asynchronous model inferences to run. The default value is 100.
for (uint32_t cnt = 0; cnt < g_executeTimes; ++cnt) {
    if (memPool->mallocMemory(input, output) != SUCCESS) {
        ERROR_LOG("get free memory failed");
        return FAILED;
    }
        // Perform asynchronous inference.
    aclError ret = aclmdlExecuteAsync(modelId_, input, output, stream_);

    if (isCallback) {
        if (dataMap == nullptr) {
            dataMap = new std::map<aclmdlDataset *, aclmdlDataset *>;
            if (dataMap == nullptr) {
                ERROR_LOG("malloc list failed, modelId is %u", modelId_);
                memPool->FreeMemory(input, output);
                return FAILED;
            }
        }
        (*dataMap)[input] = output;
        callbackCnt++;
        if ((callbackCnt % g_callbackInterval) == 0) {
                // Add a callback function to be executed to the stream.
            ret = aclrtLaunchCallback(CallBackFunc, (void *)dataMap, ACL_CALLBACK_BLOCK, stream_);
            if (ret != ACL_SUCCESS) {
                ERROR_LOG("launch callback failed, index=%zu", callbackCnt);
                memPool->FreeMemory(input, output);
                delete dataMap;
                return FAILED;
            }
            dataMap = nullptr;
        }

    }
}
//-----End of the key implementation of the user-defined ExecuteAsync function.-----

// 2.5 For asynchronous inference, the application needs to be blocked until all tasks in the specified stream are complete.
aclrtSynchronizeStream(stream_);

// 2.6 Unregister a thread so that the callback function in the stream is no longer processed by the specified thread.
aclrtUnSubscribeReport(static_cast<uint64_t>(tid), stream_);
s_isExit = true;
(void)pthread_join(tid, nullptr);

......