TGEMV

Instruction Diagram

Introduction

General matrix-vector multiplication that generates an accumulator/output tile.

Mathematical Semantics

Assume:

• M = 1
• K = bMatrix.GetValidRow()
• N = bMatrix.GetValidCol()

1. TGEMV (Tile-Based GEMV)

For 0 <= j < N (output elements in the valid matrix multiplication domain):

$$\mathrm{C}_{0,j} = \sum_{k=0}^{K-1} \mathrm{A}_{0,k} \cdot \mathrm{B}_{k,j}$$

2. TGEMV_ACC (Tile-Based GEMV with Accumulation)

For 0 <= j < N (accumulate into the existing tile):

$$\mathrm{C}_{0,j} \gets \mathrm{C}_{0,j} + \sum_{k=0}^{K-1} \mathrm{A}_{0,k} \cdot \mathrm{B}_{k,j}$$

3. TGEMV_BIAS (Tile-Based GEMV with Bias)

For 0 <= j < N (adds the bias term to the matrix product):

$$\mathrm{C}_{0,j} = \mathrm{Bias}_{0,j} + \sum_{k=0}^{K-1} \mathrm{A}_{0,k} \cdot \mathrm{B}_{k,j}$$

Note: The exact accumulator behavior and data type promotion are defined by the target/implementation.

Assembly Syntax

Synchronous form:

%acc = tgemv %a, %b : (!pto.tile<...>, !pto.tile<...>) -> !pto.tile<...>

%acc1 = tgemv.acc %acc0, %a, %b : (!pto.tile<...>, !pto.tile<...>, !pto.tile<...>) -> !pto.tile<...>

%acc = tgemv.bias %a, %b, %bias : (!pto.tile<...>, !pto.tile<...>, !pto.tile<...>) -> !pto.tile<...>

AS Level 1 (SSA)

%c = pto.tgemv %a, %b : (!pto.tile<...>, !pto.tile<...>) -> !pto.tile<...>
%c_out = pto.tgemv.acc %c_in, %a, %b : (!pto.tile<...>, !pto.tile<...>, !pto.tile<...>) -> !pto.tile<...>
%c = pto.tgemv.bias %a, %b, %bias : (!pto.tile<...>, !pto.tile<...>, !pto.tile<...>) -> !pto.tile<...>

AS Level 2 (DPS)

pto.tgemv ins(%a, %b : !pto.tile_buf<...>, !pto.tile_buf<...>) outs(%c : !pto.tile_buf<...>)
pto.tgemv.acc ins(%c_in, %a, %b : !pto.tile_buf<...>, !pto.tile_buf<...>, !pto.tile_buf<...>) outs(%c_out : !pto.tile_buf<...>)
pto.tgemv.bias ins(%a, %b, %bias : !pto.tile_buf<...>, !pto.tile_buf<...>, !pto.tile_buf<...>) outs(%c : !pto.tile_buf<...>)

C++ Built-in APIs

Declared in include/pto/common/pto_instr.hpp:

The public include header is <pto/pto-inst.hpp>, and the internal declaration is located in pto/common/pto_instr.hpp.

template <typename TileRes, typename TileLeft, typename TileRight, typename... WaitEvents>
PTO_INST RecordEvent TGEMV(TileRes &cMatrix, TileLeft &aMatrix, TileRight &bMatrix, WaitEvents&... events);

template <typename TileRes, typename TileLeft, typename TileRight, typename... WaitEvents>
PTO_INST RecordEvent TGEMV_ACC(TileRes &cOutMatrix, TileRes &cInMatrix, TileLeft &aMatrix, TileRight &bMatrix, WaitEvents&... events);

template <typename TileRes, typename TileLeft, typename TileRight, typename TileBias, typename... WaitEvents>
PTO_INST RecordEvent TGEMV_BIAS(TileRes &cMatrix, TileLeft &aMatrix, TileRight &bMatrix, TileBias &biasData, WaitEvents&... events);

Constraints

General Shape and Position Constraints

Unless otherwise specified, the following constraints apply to TGEMV, TGEMV_ACC, and TGEMV_BIAS.

• Static shape constraints:
  • TileLeft::Rows == TileRes::Rows
  • TileLeft::Cols == TileRight::Rows
  • TileRight::Cols == TileRes::Cols
• Tile position constraints:
  • TileLeft::Loc == Left
  • TileRight::Loc == Right
  • TileRes::Loc == Acc
• Runtime valid size constraints:
  • m must be 1
  • k and n (obtained from bMatrix.GetValidRow() and bMatrix.GetValidCol()) must be within [1, 4095]

TGEMV/TGEMV_ACC Data Type Constraints

• Implementation check (Atlas A2/A3 training products/Atlas A2/A3 inference products):
  • Supported (CType, AType, BType) triples:
    • (int32_t, int8_t, int8_t)
    • (float, half, half)
    • (float, float, float)
    • (float, bfloat16_t, bfloat16_t)
• Implementation check (Ascend 950PR/Ascend 950DT):
  • The accumulator type must be int32_t or float.
  • If it is int32_t: AType == int8_t and BType == int8_t.
  • If it is float: supports half, bfloat16_t, float, selected fp8 combinations, and hifloat8_t/hifloat8_t (target-defined).
  • The following fractal/layout constraints are enforced:
    • Left: Loc == Left, !isRowMajor, SFractal == RowMajor
    • Right: Loc == Right, isRowMajor, SFractal == ColMajor
    • Acc: Loc == Acc, !isRowMajor, SFractal == RowMajor

Additional Constraints for TGEMV_BIAS

• The bias tile data type must be exactly the same as TileRes::DType.
• The bias tile must be configured as a single row.
• The bias tile position must be TileType::Bias.
• Additional notes for Ascend 950PR/Ascend 950DT:
  • In addition to the GEMV conventions above, the underlying Ascend 950PR/Ascend 950DT matmul implementation does not add a separate set of explicit m/k/n runtime assertions.

Examples

Automatic

1. TGEMV

#include <pto/pto-inst.hpp>

using namespace pto;

void example_auto() {
  using A = TileLeft<half, 1, 16>;
  using B = TileRight<half, 16, 16>;
  using C = TileAcc<float, 1, 16>;
  A a;
  B b;
  C c;
  TGEMV(c, a, b);
}

2. TGEMV_ACC

#include <pto/pto-inst.hpp>

using namespace pto;

void example_auto() {
  using A = TileLeft<half, 1, 16>;
  using B = TileRight<half, 16, 16>;
  using C = TileAcc<float, 1, 16>;
  A a;
  B b;
  C c0, c1;
  TGEMV_ACC(c1, c0, a, b);
}

3. TGEMV_BIAS

#include <pto/pto-inst.hpp>

using namespace pto;

void example_auto() {
  using A = TileLeft<half, 1, 16>;
  using B = TileRight<half, 16, 16>;
  using Bias = Tile<TileType::Bias, float, 1, 16>;
  using C = TileAcc<float, 1, 16>;
  A a;
  B b;
  Bias bias;
  C c;
  TGEMV_BIAS(c, a, b, bias);
}

Manual

1. TGEMV

#include <pto/pto-inst.hpp>

using namespace pto;

void example_manual() {
  using A = TileLeft<half, 1, 16>;
  using B = TileRight<half, 16, 16>;
  using C = TileAcc<float, 1, 16>;
  A a;
  B b;
  C c;
  TASSIGN(a, 0x1000);
  TASSIGN(b, 0x2000);
  TASSIGN(c, 0x3000);
  TGEMV(c, a, b);
}

2. TGEMV_ACC

#include <pto/pto-inst.hpp>

using namespace pto;

void example_manual() {
  using A = TileLeft<half, 1, 16>;
  using B = TileRight<half, 16, 16>;
  using C = TileAcc<float, 1, 16>;
  A a;
  B b;
  C c0, c1;
  TASSIGN(a, 0x1000);
  TASSIGN(b, 0x2000);
  TASSIGN(c0, 0x3000);
  TASSIGN(c1, 0x4000);
  TGEMV_ACC(c1, c0, a, b);
}

3. TGEMV_BIAS

#include <pto/pto-inst.hpp>

using namespace pto;

void example_manual() {
  using A = TileLeft<half, 1, 16>;
  using B = TileRight<half, 16, 16>;
  using Bias = Tile<TileType::Bias, float, 1, 16>;
  using C = TileAcc<float, 1, 16>;
  A a;
  B b;
  Bias bias;
  C c;
  TASSIGN(a, 0x1000);
  TASSIGN(b, 0x2000);
  TASSIGN(bias, 0x3000);
  TASSIGN(c, 0x4000);
  TGEMV_BIAS(c, a, b, bias);
}

ASM Examples

Automatic Mode

# Automatic mode: the compiler/runtime handles resource placement and scheduling.
%c = pto.tgemv %a, %b : (!pto.tile<...>, !pto.tile<...>) -> !pto.tile<...>

Manual Mode

# Manual mode: explicitly bind resources first, then issue the instruction.
# Optional (when the instruction contains tile operands):
# pto.tassign %arg0, @tile(0x1000)
# pto.tassign %arg1, @tile(0x2000)
%c = pto.tgemv %a, %b : (!pto.tile<...>, !pto.tile<...>) -> !pto.tile<...>

PTO Assembly Form

%acc = tgemv %a, %b : (!pto.tile<...>, !pto.tile<...>) -> !pto.tile<...>
# AS Level 2 (DPS)
pto.tgemv ins(%a, %b : !pto.tile_buf<...>, !pto.tile_buf<...>) outs(%c : !pto.tile_buf<...>)