Asynchronous Model Inference

#include "acl/acl.h"
// ......
// 1. Initialize AscendCL.

// 2. Allocate runtime resources.

// Obtain the run mode of the AI software stack. The API calls for memory allocation and memory copy are different in different run modes.
aclrtRunMode runMode;
extern bool g_isDevice;
aclrtGetRunMode(&runMode);
g_isDevice = (runMode == ACL_DEVICE);

// 3. Allocate model inference resources.

// The two dots (..) indicate a path relative to the directory of the executable file.
// For example, if the executable file is stored in the out directory, the two dots (..) point to the parent directory of the out directory.
const char* omModelPath = "../model/resnet50.om";

// 3.1 Load a model.
// Obtain the weight memory size and workspace size required for model execution, and allocate memory as required.
aclmdlQuerySize(omModelPath, &modelMemSize_, &modelWeightSize_);
aclrtMalloc(&modelMemPtr_, modelMemSize_, ACL_MEM_MALLOC_HUGE_FIRST);
aclrtMalloc(&modelWeightPtr_, modelWeightSize_, ACL_MEM_MALLOC_HUGE_FIRST);

// Load the offline model file. After the model is successfully loaded, the model ID is returned.
aclmdlLoadFromFileWithMem(modelPath, &modelId_, modelMemPtr_,
        modelMemSize_, modelWeightPtr_, modelWeightSize_);

// 3.2 Obtain the model description based on the model ID.
modelDesc_ = aclmdlCreateDesc();
aclmdlGetDesc(modelDesc_, modelId_);

// 3.3 Customize the InitMemPool function for memory pool initialization, to store the input and output data for model inference.
// -----Key implementation of the InitMemPool function.-----
string testFile[] = {
        "../data/dog1_1024_683.bin",
        "../data/dog2_1024_683.bin"
    };
size_t fileNum = sizeof(testFile) / sizeof(testFile[0]);
// g_memoryPoolSize indicates the number of memory blocks in the memory pool. The default value is 100.
for (size_t i = 0; i < g_memoryPoolSize; ++i) {
        size_t index = i % (sizeof(testFile) / sizeof(testFile[0]));
        // model process
        uint32_t devBufferSize;
        // Customize the GetDeviceBufferOfFile function to implement the following functions:
        // Obtain the buffer for storing the input image data together with the buffer size, and transfer the image data to the device.
        void *picDevBuffer = Utils::GetDeviceBufferOfFile(testFile[index], devBufferSize);
        aclmdlDataset *input = nullptr;
// Customize the CreateInput function to create data input of type aclmdlDataset for storing the input data of model inference.
        Result ret = CreateInput(picDevBuffer, devBufferSize, input);
        aclmdlDataset *output = nullptr;
        // Customize the CreateOutput function to create data output of type aclmdlDataset for storing the output data of model inference. modelDesc indicates the model description.
        CreateOutput(output, modelDesc);
        {
            std::lock_guard<std::recursive_mutex> lk(freePoolMutex_);
            freeMemoryPool_[input] = output;
        }
}
// -----Key implementation of the InitMemPool function.-----

// 4. Model inference
// 4.1 Create a thread tid and subscribe to thread tid for handling the callback function in the stream.
// ProcessCallback is a thread function. Call aclrtProcessReport in the thread function, and the callback function is triggered after a specified period of time.
pthread_t tid;
(void)pthread_create(&tid, nullptr, ProcessCallback, &s_isExit);

// 4.2 Subscribe to a thread for handling the callback function in the stream.
aclrtSubscribeReport(tid, stream_);

// 4.2 Create a callback function for processing the model inference result. The callback function is user-defined.
void ModelProcess::CallBackFunc(void *arg)
{
    std::map<aclmdlDataset *, aclmdlDataset *> *dataMap =
        (std::map<aclmdlDataset *, aclmdlDataset *> *)arg;

    aclmdlDataset *input = nullptr;
    aclmdlDataset *output = nullptr;
    MemoryPool *memPool = MemoryPool::Instance();

    for (auto& data : *dataMap) {
        ModelProcess::OutputModelResult(data.second);
        memPool->FreeMemory(data.first, data.second);
    }

    delete dataMap;
}

// 4.3 Customize the ExecuteAsync function to perform model inference.
//-----Start of the key implementation of the user-defined ExecuteAsync function.-----
// g_callbackInterval indicates the callback interval. The default value is 1, indicating that a callback task is delivered every asynchronous inference.
bool isCallback = (g_callbackInterval != 0);
    size_t callbackCnt = 0;
    std::map<aclmdlDataset *, aclmdlDataset *> *dataMap = nullptr;
    aclmdlDataset *input = nullptr;
    aclmdlDataset *output = nullptr;
    MemoryPool *memPool = MemoryPool::Instance();
    // g_executeTimes indicates the number asynchronous model inferences to run. The default value is 100.
    for (uint32_t cnt = 0; cnt < g_executeTimes; ++cnt) {
        if (memPool->mallocMemory(input, output) != SUCCESS) {
            ERROR_LOG("get free memory failed");
            return FAILED;
        }
        // Perform asynchronous inference.
        aclError ret = aclmdlExecuteAsync(modelId_, input, output, stream_);

        if (isCallback) {
            if (dataMap == nullptr) {
                dataMap = new std::map<aclmdlDataset *, aclmdlDataset *>;
                if (dataMap == nullptr) {
                    ERROR_LOG("malloc list failed, modelId is %u", modelId_);
                    memPool->FreeMemory(input, output);
                    return FAILED;
                }
            }
            (*dataMap)[input] = output;
            callbackCnt++;
            if ((callbackCnt % g_callbackInterval) == 0) {
                // Add a callback function to be executed to the stream.
                ret = aclrtLaunchCallback(CallBackFunc, (void *)dataMap, ACL_CALLBACK_BLOCK, stream_);
                if (ret != ACL_SUCCESS) {
                    ERROR_LOG("launch callback failed, index=%zu", callbackCnt);
                    memPool->FreeMemory(input, output);
                    delete dataMap;
                    return FAILED;
                }
                dataMap = nullptr;
            }

        }
    }
//-----End of the key implementation of the user-defined ExecuteAsync function.-----

// 4.4 For asynchronous inference, block app execution until the stream has completed all preceding requested tasks.
aclrtSynchronizeStream(stream_);

// 4.5 Unsubscribe from a thread and specify a callback function in a stream to not be handled by a specified thread.
aclrtUnSubscribeReport(static_cast<uint64_t>(tid), stream_);
s_isExit = true;
(void)pthread_join(tid, nullptr);

// 5. Deallocate runtime resources.

// 6. Deinitialize AscendCL.

// ......