Precautions for Scope Fusion in the Many-to-Many Scenario

This includes the development process, fusion result setting, and information processing required by internal small operators.

Overview

In the many-to-many scenario, multiple small operators in the scope are fused into a composite of small operators.

Procedure

For details about the implementation process, see Many-to-Many Scope Fusion. The many-to-many scope fusion differs from traditional scope fusion in the following aspects:

In the fusion result setting function GenerateFusionResult, you need to set the connections between internal small operator composites.
The internal small operators are not parsed by the plugin. Therefore, you need to construct IRs for the small operators and set required parameters such as attributes.

Setting the Fusion Result

To ensure correct logic, the Identity operator is added before the input and output of the DecodeBboxV2 operator in the example code. In this way, more operators can be added. In practice, you can add small operator composites based on the specific operator function requirements.

Take the following steps:

Set the name, description, and external input and output of the fusion result, which are the same as those in the many-to-one scenario.
Set the fusion operator type to a specific type by referring to kScopeToMultiNodes, indicating that the fusion operators are fused into multiple operator composites.
```
fusion_rlt->SetType(kScopeToMultiNodes);  // Set the special fusion operator type, indicating that the operators are fused into multiple operator composites.
```

Set the name, type, attribute, input, and output of a small operator by referring to AddInnerNode.

Figure 1 Adding small fusion operators

For example:

  auto in_identity_1 = fusion_rlt->AddInnerNode("input_identity_1", "Identity");
  CHECK_INNER_NODE_CONDITION(in_identity_1 != nullptr, fusion_rlt);
  ret = in_identity_1->InsertInput(kInputFromFusionScope, 1) // Input 1 from the fusion result boundary.
    .InsertOutput("inner_core_decode_bbox_v2", 1)               // Output to internal operators.
    .BuildInnerNode();
  CHECK_INNER_NODE_CONDITION(ret == ge::GRAPH_SUCCESS, fusion_rlt);

kInputFromFusionScope: indicates that the input of the target small operator comes from the scope boundary. kOutputToFusionScope: indicates that the output of the target small operator is output to the scope boundary.
InsertInput: sets the input of the target small operator.
InsertOutput: sets the output of the target small operator.
BuildInnerNode: builds internal small operators.

The following is a complete code example.

void DecodeBboxV2MultiScopeFusionPass::GenerateFusionResult(const std::vector<Scope *> &scopes, FusionScopesResult *fusion_rlt) {
  if (fusion_rlt == nullptr) {
    return;
  }
  if (scopes.size() != 1) {
    fusion_rlt->SetType(kScopeInvalidType);
    return;
  }

  // Set the input of the fusion result. Use input 0 of the transpose operator as input 0 of the fusion result. Input 1 of the transpose operator is not used.
  fusion_rlt->InsertInputs("transpose", {0, kFusionDisableIndex});
  // Set the input of the fusion result. Use input 0 of get_center_coordinates_and_sizes/transpose as input 1 of the fusion result. Input 1 of get_center_coordinates_and_sizes/transpose is not used.
  fusion_rlt->InsertInputs("get_center_coordinates_and_sizes/transpose", {1, kFusionDisableIndex});
  // Set the output of the fusion result. Use output 0 of transpose_1 as the output of the fusion result.
  fusion_rlt->InsertOutputs("transpose_1", {0});

fusion_rlt->SetType(kScopeToMultiNodes);  // Set the special fusion operator type, indicating that the operators are fused into multiple operator composites.
  AscendString scope_name;
  Status ret = scopes[0]->Name(scope_name);
  if (ret != SUCCESS) {
    return ;
  }
  std::string str_scope_name;
  if (scope_name != nullptr) {
    str_scope_name = scope_name.GetString();
  }
  fusion_rlt->SetName(str_scope_name.substr(0, str_scope_name.length() - 1).c_str());
  fusion_rlt->SetDescription("");

// Add small fusion operators.
  auto in_identity_0 = fusion_rlt->AddInnerNode("input_identity_0", "Identity");
  CHECK_INNER_NODE_CONDITION(in_identity_0 != nullptr, fusion_rlt);
  Status ret = in_identity_0->InsertInput(kInputFromFusionScope, 0)  // Input 0 from the fusion result boundary.
    .InsertOutput("inner_core_decode_bbox_v2", 0)                    //  Output to internal small operators.
    .BuildInnerNode();
  CHECK_INNER_NODE_CONDITION(ret == ge::GRAPH_SUCCESS, fusion_rlt);
  std::string str_attr = "input_0_identity_attr";
  in_identity_0->MutableOperator()->SetAttr("key", str_attr);

  auto in_identity_1 = fusion_rlt->AddInnerNode("input_identity_1", "Identity");
  CHECK_INNER_NODE_CONDITION(in_identity_1 != nullptr, fusion_rlt);
  ret = in_identity_1->InsertInput(kInputFromFusionScope, 1) // Input 1 from the fusion result boundary.
    .InsertOutput("inner_core_decode_bbox_v2", 1)            //Output to internal small operators.
    .BuildInnerNode();
  CHECK_INNER_NODE_CONDITION(ret == ge::GRAPH_SUCCESS, fusion_rlt);

  auto core_decode_bbox = fusion_rlt->AddInnerNode("inner_core_decode_bbox_v2", kScopeType);
  CHECK_INNER_NODE_CONDITION(core_decode_bbox != nullptr, fusion_rlt);
  ret = core_decode_bbox->InsertInput("input_identity_0", 0)
    .InsertInput("input_identity_1", 0)
    .InsertOutput("output_identity", 0)
    .BuildInnerNode();
  CHECK_INNER_NODE_CONDITION(ret == ge::GRAPH_SUCCESS, fusion_rlt);
  // Set the parameters of the small operator after fusion as required.
  auto parser_ret = DecodeBboxV2ParseParams(fusion_rlt->Nodes(), core_decode_bbox->MutableOperator());
  CHECK_INNER_NODE_CONDITION(parser_ret == SUCCESS, fusion_rlt);

  auto out_identity = fusion_rlt->AddInnerNode("output_identity", "Identity");
  CHECK_INNER_NODE_CONDITION(out_identity != nullptr, fusion_rlt);
  ret = out_identity->InsertInput("inner_core_decode_bbox_v2", 0) // Output 0 from internal small operators.
    .InsertOutput(kOutputToFusionScope, 0)                        // Output 0 to the fusion result boundary.
    .BuildInnerNode();
  CHECK_INNER_NODE_CONDITION(ret == ge::GRAPH_SUCCESS, fusion_rlt);

  ret = fusion_rlt->CheckInnerNodesInfo();
  CHECK_INNER_NODE_CONDITION(ret == ge::GRAPH_SUCCESS, fusion_rlt);

  OP_LOGI(kOpType, "Set fusion multi-to-multi result successfully.");
  return;
}

Information Processing Required by Internal Small Operators

Some attributes may need to be set for small operators inside the fusion result. The small operators are not parsed by the plugin. Therefore, the attributes need to be added when the small operators are constructed. In the example, the target small operator DecodeBboxV2 needs to obtain the scale information based on the small operator of the original graph scope.

namespace {
Status ParseFloatFromConstNode(const ge::OperatorPtr node, float &value) {
  if (node == nullptr) {
    return FAILED;
  }
  ge::Tensor tensor;
  auto ret = node->GetAttr("value", tensor);
  if (ret != ge::GRAPH_SUCCESS) {
    AscendString op_name;
    graphStatus ret = node->GetName(op_name);
    if (ret != ge::GRAPH_SUCCESS) {
      return FAILED;
    }
    OP_LOGE(kOpType, "Failed to get value from %s", op_name.GetString());
    return FAILED;
  }
  uint8_t *data_addr = tensor.GetData();
  value = *(reinterpret_cast<float *>(data_addr));
  return SUCCESS;
}

Status DecodeBboxV2ParseParams(const std::vector<ge::OperatorPtr> &inside_nodes, ge::Operator *op_dest) {
  if (op_dest == nullptr) {
    OP_LOGE(kOpType, "Dest operator is nullptr.");
    return FAILED;
  }
  std::map<std::string, std::string> scales_const_name_map;
  std::map<string, ge::OperatorPtr> node_map;
  for (const auto &node : inside_nodes) {
    if (node == nullptr) {
      OP_LOGE(kOpType, "Inner operator is nullptr.");
      return FAILED;
    }
    ge::AscendString op_type;
    ge::graphStatus ret = node.GetOpType(op_type);
    if (ret != ge::GRAPH_SUCCESS) {
      return FAILED;
    }
    ge::AscendString op_name;
    ret = node.GetName(op_name);
    string str_op_name;
    if (op_name.GetString() != nullptr) {
      str_op_name = op_name.GetString();
    }
    if (op_type == kBoxesDiv) {
      if (node->GetInputsSize() < kRealDivInputSize) {
        OP_LOGE(kOpType, "Input size of %s is invalid, which is %zu.", kBoxesDiv, node->GetInputsSize());
        return FAILED;
      }
      ge::AscendString input_unpack_name0;
      ret = node.GetInputDesc(0).GetName(input_unpack_name0);
      string str_input_unpack_name0;
      if (input_unpack_name0.GetString() != nullptr) {
        str_input_unpack_name0 = input_unpack_name0.GetString();
      }
      ge::AscendString input_unpack_name1;
      ret = node.GetInputDesc(1).GetName(input_unpack_name1);
      string str_input_unpack_name1;
      if (input_unpack_name1.GetString() != nullptr) {
        str_input_unpack_name1 = input_unpack_name1.GetString();
      }
      if (str_input_unpack_name0.find(kBoxesUnpack) != string::npos) {
        scales_const_name_map.insert({str_op_name, str_input_unpack_name1 });
      }
    }
    node_map[str_op_name] = &node;
  }

  std::vector<float> scales_list = {1.0, 1.0, 1.0, 1.0};
  if (scales_const_name_map.size() != kScaleSize) {
    OP_LOGI(op_dest.GetName().c_str(), "Boxes doesn't need scale.");
  } else {
    size_t i = 0;
    for (const auto &name_pair : scales_const_name_map) {
      float scale_value = 1.0;
      auto ret = ParseFloatFromConstNode(node_map[name_pair.second], scale_value);
      if (ret != SUCCESS) {
        return ret;
      }
      scales_list[i++] = scale_value;
    }
  }
  op_dest->SetAttr("scales", scales_list);
  return SUCCESS;
}
}  // namespace

Parent topic: Developing Fusion Patterns