Introduction to Stream-Ordered Memory Allocation

Background

aclrtMalloc and aclrtFree are used to allocate and manage memory synchronously. The sample code below shows a common memory usage scenario: Use aclrtMalloc to allocate memory, and copy the memory data to the device asynchronously for operators to use. After the operator execution is complete, use stream synchronization to confirm that the memory is used up, and then use aclrtFree to free the memory. This method is acceptable when there are few asynchronous operations. However, when there are a large number of delivered operators and frequent asynchronous operations on the allocated memory, this method has the following disadvantages:

Synchronization bottleneck: When a task is being delivered, if memory allocation or freeing needs to be adjusted, a synchronization bottleneck may occur, affecting the overall efficiency.
Accumulated latency: Memory allocation and freeing are time-consuming. Frequent operations will accumulate unnecessary latency, further degrading performance.

#include "acl/acl_rt.h"
#include "acl/acl.h"

int main() {
    typedef struct {
        ......
    } ArgsInfo;

    void *ptr0 = nullptr;
    aclrtStream stream1;
    aclrtCreateStream(&stream1);

    // Allocate memory.
    aclrtMalloc(&ptr0, sizeof(ArgsInfo), ACL_MEM_MALLOC_HUGE_FIRST);
    ......

    // Configuration task delivery.
    ArgsInfo usrArgs;
    // Copy information to the device to allocate memory.
    error = aclrtMemcpyAsync(ptr0, sizeof(ArgsInfo), (void *)&usrArgs, sizeof(ArgsInfo), ACL_MEMCPY_HOST_TO_DEVICE, stream1);

    // Deliver a task.
    uint32_t blockDim = 32;
    aclrtLaunchKernelV2(funcHandle, blockDim, (void *)&usrArgs, sizeof(ArgsInfo), nullptr, stream1);

    // Perform stream synchronization to free the allocated memory.
    aclrtSynchronizeStream(stream1);

    // Free the memory. Stream synchronization is required before memory freeing.
    aclrtFree(ptr0);
    ......

    // Perform stream synchronization.
    aclrtSynchronizeStream(stream1);

    return 0;
}

In contrast, the stream-ordered memory allocation described in this section integrates memory allocation and deallocation operations into the stream scheduling sequence to manage memory. This approach tightly couples memory management with task execution in streams, allowing memory management without explicit synchronization of tasks in a stream. Additionally, stream-ordered execution ensures that operations are executed in a well-defined order. In addition, the Runtime provides memory reuse to support complex memory management scenarios.

#include "acl/acl_rt.h"
#include "acl/acl.h"

int main() {
    typedef struct {
        ......
    } ArgsInfo;

    void *ptr0 = nullptr;
    aclrtStream stream1;
    aclrtCreateStream(&stream1);

    // Set memory pool attributes.
    aclrtMemLocation testLoc = {
            0,                           // id
            ACL_MEM_LOCATION_TYPE_DEVICE // type
    };
    aclrtMemPoolProps testProp = {
            ACL_MEM_ALLOCATION_TYPE_PINNED, // allocType
            ACL_MEM_HANDLE_TYPE_NONE,       // handleType
            testLoc,                        // location
            14UL << 30, // maxSize = 14 GB. The memory pool size is 14 GB.
            {0}                             // reserved
    };

    // Create a memory pool.
    aclrtMemPool testReusePool;
    auto ret = aclrtMemPoolCreate(&testReusePool, &testProp);
    if (ret != ACL_SUCCESS) {
        fprintf(stderr, "Failed to create memory pool\n");
        return -1;
    }

    // Asynchronously allocate memory. testReusePool is the memory pool created by the user.
    aclrtMemPoolMallocAsync(&ptr0, sizeof(ArgsInfo), testReusePool, stream1);
    ......

    // Configuration task delivery.
    ArgsInfo usrArgs;
    // Copy information to the device to allocate memory.
    error = aclrtMemcpyAsync(ptr0, sizeof(ArgsInfo), (void *)&usrArgs, sizeof(ArgsInfo), ACL_MEMCPY_HOST_TO_DEVICE, stream1);

    // Deliver a task.
    uint32_t blockDim = 32;
    aclrtLaunchKernelV2(funcHandle, blockDim, (void *)&usrArgs, sizeof(ArgsInfo), nullptr, stream1);

    // Asynchronously free memory, without stream synchronization.
    aclrtMemPoolFreeAsync(ptr0, stream1);
    ......

    // Perform stream synchronization.
    aclrtSynchronizeStream(stream1);

    return 0;
}

Memory Reuse

When aclrtMemPoolFreeAsync is called, the memory is returned to the memory pool and is not physically freed. In this way, subsequent tasks can reuse the physical memory, avoiding frequent allocation and deallocation of physical memory and improving performance. During memory reuse, the free memory with the size closest to the required memory size of the current task is selected.

By default, when the free physical memory in the memory pool exceeds the specified threshold (which can be configured using the aclrtMemPoolSetAttr API and defaults to 0), the system attempts to release the free physical memory during the next stream synchronization (for example, when the aclrtSynchronizeStream API is called). The Runtime also provides the aclrtMemPoolTrimTo API, which can be directly called to proactively trim the memory pool and release free physical memory. If you have configured the threshold for the size of the memory to be retained in the memory pool (corresponding to the ACL_RT_MEM_POOL_ATTR_RELEASE_THRESHOLD configuration item) by calling the aclrtMemPoolSetAttr API and also configured the size of the physical memory to be retained by setting the minBytesToKeep parameter in the aclrtMemPoolTrimTo API, the latter has a higher priority.

Currently, memory can be reused in one stream or between two streams.

When memory is reused in a stream, the following mechanism is used: When executing a task in a stream, the system searches for the memory that has been returned to the memory pool by the previous tasks in the stream and reuses the memory to improve memory utilization and reduce memory allocation overhead.
When memory is reused between two streams:
- Event-based memory reuse: During task execution in a stream, the system searches for other streams associated with this stream through events and reuses the memory previously returned to the memory pool by tasks in those associated streams. This mechanism applies to scenarios where tasks between streams are synchronized through events in user applications.
- Opportunistic memory reuse: During task execution in a stream, the system searches for reusable memory in the memory pool, but does not guarantee successful memory reuse. If memory reuse fails, the program reports an error and stops.
- Implicit dependency-based memory reuse: During task execution in a stream, the system searches for reusable memory in the memory pool. If the memory was previously used by another stream but no task dependencies exist between the two streams, the system automatically performs synchronization and wait between the streams. This ensures that the previous stream has finished accessing the memory, thereby achieving secure memory reuse.

Sample Code

The sample code below demonstrates the scenario for asynchronous memory allocation and deallocation, and uses the aclrtLaunchKernelV2 API to deliver tasks. The code is for reference only and cannot be directly copied for compilation. Adjust the code based on the actual environment and requirements.

#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <time.h>
#include <gtest/gtest.h>
#include "acl/acl_rt.h"
#include "acl/acl.h"

int main() {
    uint32_t devid = 0;
    aclInit(NULL);
    aclrtSetDevice(devid);

    // Create a context and a stream.
    aclrtContext context;
    aclrtStream stream1;

    aclrtCreateContext(&context, 0);
    aclrtCreateStream(&stream1);

    // Set memory pool attributes.
    aclrtMemLocation testLoc = {
            0,                           // id
            ACL_MEM_LOCATION_TYPE_DEVICE // type
    };
    aclrtMemPoolProps testProp = {
            ACL_MEM_ALLOCATION_TYPE_PINNED, // allocType
            ACL_MEM_HANDLE_TYPE_NONE,       // handleType
            testLoc,                        // location
            14UL << 30, // maxSize = 14 GB. The memory pool size is 14 GB.
            {0}                             // reserved
    };

    // Create a memory pool.
    aclrtMemPool testReusePool;
    auto ret = aclrtMemPoolCreate(&testReusePool, &testProp);
    if (ret != ACL_SUCCESS) {
        fprintf(stderr, "Failed to create memory pool\n");
        return -1;
    }

    const size_t GB_TO_B = 1024ULL * 1024 * 1024;

    // Define the operator information structure.
    typedef struct {
        void *input_x;
        void *input_y;
        void *output_z;
    } ArgsInfo;

    aclrtBinHandle binHandle = nullptr;
    aclrtFuncHandle funcHandle;
    aclError aclrtBinaryGetFunction(binHandle, "add_custom", &funcHandle);

    void *ptr0 = nullptr;
    void *ptr1 = nullptr;
    void *ptr2 = nullptr;
    void *ptr3 = nullptr;

    // Allocate memory asynchronously.
    aclrtMemPoolMallocAsync(&ptr1, sizeof(uint64_t), testReusePool, stream1);
    aclrtMemPoolMallocAsync(&ptr2, sizeof(uint64_t), testReusePool, stream1);
    aclrtMemPoolMallocAsync(&ptr3, sizeof(uint64_t), testReusePool, stream1);
    aclrtMemPoolMallocAsync(&ptr0, sizeof(ArgsInfo), testReusePool, stream1);

    // Configuration task delivery.
    ArgsInfo usrArgs;
    usrArgs.input_x = ptr1
    usrArgs.input_y = ptr2;
    usrArgs.output_z = ptr3;

    error = aclrtMemcpyAsync(ptr0, sizeof(ArgsInfo), (void *)&usrArgs, sizeof(ArgsInfo), ACL_MEMCPY_HOST_TO_DEVICE, stream1);

    // Deliver a task.
    uint32_t blockDim = 32;
    aclrtLaunchKernelV2(funcHandle, blockDim, (void *)&usrArgs, sizeof(ArgsInfo), nullptr, stream1);

    // Asynchronously free memory, without stream synchronization.
    aclrtMemPoolFreeAsync(ptr0, stream1);
    aclrtMemPoolFreeAsync(ptr1, stream1);
    aclrtMemPoolFreeAsync(ptr2, stream1);
    aclrtMemPoolFreeAsync(ptr3, stream1);

    // Perform stream synchronization.
    aclrtSynchronizeStream(stream1);

    // Destroy the memory pool, stream, and context.
    aclrtMemPoolDestroy(testReusePool);
    aclrtDestroyStream(stream1);
    aclrtDestroyContext(context);

    aclrtResetDevice(devid);
    aclFinalize();
    return 0;
}

Parent topic: Stream-Ordered Memory Allocation