在MindSpore GPU中推理两个等价模型,输出不一致
收藏回复举报
在MindSpore GPU中推理两个等价模型,输出不一致
发表于2025-04-29 10:22:51
0 查看

1 系统环境

硬件环境(Ascend/GPU/CPU): GPU

MindSpore版本: mindspore=2.5.0

执行模式(PyNative/ Graph): PyNative

Python版本: Python=3.10

操作系统平台: Ubuntu22.04

2 报错信息

2.1 问题描述

在MindSpore GPU模式下,两个数学上等价的模型(涉及uint8→int8的Cast、Where、Max等操作)推理结果出现不一致。理论上两个模型输出应完全一致,但实际Max结果有0.2%的元素不匹配,最大差异高达251。而Less操作输出完全一致,表明问题出现在Max操作在Cast前后的位置差异导致的整型数据流异常。

定义两个数学上等价的模型:

T1Model 执行顺序:Where → Max → Cast → Min → Less
T2Model 执行顺序:Where → Cast → Min → Less → Max

2.2 推理脚本

import numpy as np
import mindspore as ms
import mindspore.nn as nn
import mindspore.ops as ops

ms.set_context(mode=ms.PYNATIVE_MODE, device_target="GPU")
np.random.seed(42)

# 创建输入数据
mask = ms.Tensor(np.random.choice([True, False], size=(5, 14, 1, 14, 57)))
val1 = ms.Tensor(np.random.randint(0, 255, size=(57,), dtype=np.uint8))
val2 = ms.Tensor(np.random.randint(0, 255, size=(1, 1, 57), dtype=np.uint8))

# 执行两种模型
t1_max, t1_less = T1Model()(mask, val1, val2)
t2_max, t2_less = T2Model()(mask, val1, val2)

2.3 结果信息

== T1模型输出 ==
Max结果 - 形状: (5, 14, 1, 14, 57) 数据类型: UInt8
部分值 (前5个元素): [135  75 133 120  34]
统计信息 - 最小值: 0 最大值: 253

== T2模型输出 ==
Max结果 - 形状: (5, 14, 1, 14, 57) 数据类型: UInt8
部分值 (前5个元素): [135  75 133 120  34]
统计信息 - 最小值: 0 最大值: 253

== Max算子结果比较 ==
不匹配元素数量: 92 / 55860 (0.2%)
最大绝对差异: 251
最大差异位置(0, 0, 0, 9, 42): T1=0, T2=5

== Less算子结果比较 ==
Less输出完全一致

2.4 脚本信息

理论上两个模型输出应完全一致,但实际Max结果有0.2%的元素不匹配,最大差异高达251。而Less结果完全一致,表明问题出现在Max操作的位置不同导致的整型数据流异常。

import numpy as np
import mindspore as ms
import mindspore.nn as nn
import mindspore.ops as ops

class T1Model(nn.Cell):
    def __init__(self):
        super(T1Model, self).__init__()
        self.cast = ops.Cast()
        
    def construct(self, v5_0, v6_0, v7_0):
        # v5_0: b[5, 14, 1, 14, 57], v6_0: u8[57], v7_0: u8[1, 1, 57]
        v8_0 = ops.where(v5_0, v6_0, v7_0)           # core.Where
        v3_0 = ops.maximum(v8_0, v8_0)               # core.Max
        v1_0 = self.cast(v8_0, ms.int8)              # Cast to int8
        v2_0 = ops.minimum(v1_0, v1_0)               # core.Min
        v4_0 = ops.less(v2_0, v1_0)                  # core.Less
        
        # 返回多个结果方便分析
        return v3_0, v4_0

class T2Model(nn.Cell):
    def __init__(self):
        super(T2Model, self).__init__()
        self.cast = ops.Cast()
        
    def construct(self, v2_0, v1_0, v0_0):
        # v2_0: b[5, 14, 1, 14, 57], v1_0: u8[57], v0_0: u8[1, 1, 57]  
        v3_0 = ops.where(v2_0, v1_0, v0_0)           # core.Where
        v4_0 = self.cast(v3_0, ms.int8)              # Cast to int8
        v5_0 = ops.minimum(v4_0, v4_0)               # core.Min  
        v6_0 = ops.less(v5_0, v4_0)                  # core.Less
        v8_0 = ops.maximum(v3_0, v3_0)               # core.Max
        
        # 返回多个结果方便分析
        return v8_0, v6_0

def reproduce_bug():
    # 设置环境
    ms.set_context(mode=ms.PYNATIVE_MODE)
    ms.set_device("GPU")
    
    # 创建模型
    t1 = T1Model()
    t2 = T2Model()
    
    # 根据图创建输入 - 严格按照DOT文件中的形状和类型
    np.random.seed(42)
    shape1 = (5, 14, 1, 14, 57)
    shape2 = (57,)
    shape3 = (1, 1, 57)
    
    # T1输入
    v5_0_t1 = ms.Tensor(np.random.choice([True, False], size=shape1))
    v6_0_t1 = ms.Tensor(np.random.randint(0, 255, shape2, dtype=np.uint8))
    v7_0_t1 = ms.Tensor(np.random.randint(0, 255, shape3, dtype=np.uint8))
    
    # T2输入 - 保持相同输入以便比较
    v2_0_t2 = v5_0_t1  # 布尔掩码
    v1_0_t2 = v6_0_t1  # uint8[57]
    v0_0_t2 = v7_0_t1  # uint8[1, 1, 57]
    
    # 执行模型
    t1_v3_0, t1_v4_0 = t1(v5_0_t1, v6_0_t1, v7_0_t1)
    t2_v8_0, t2_v6_0 = t2(v2_0_t2, v1_0_t2, v0_0_t2)
    
    # 打印T1和T2的输出
    print("\n== T1模型输出 ==")
    print("Max结果 - 形状:", t1_v3_0.shape, "数据类型:", t1_v3_0.dtype)
    print("部分值 (前5个元素):", t1_v3_0.asnumpy().flatten()[:5])
    print("统计信息 - 最小值:", np.min(t1_v3_0.asnumpy()), "最大值:", np.max(t1_v3_0.asnumpy()))

    print("\n== T2模型输出 ==")
    print("Max结果 - 形状:", t2_v8_0.shape, "数据类型:", t2_v8_0.dtype)
    print("部分值 (前5个元素):", t2_v8_0.asnumpy().flatten()[:5])
    print("统计信息 - 最小值:", np.min(t2_v8_0.asnumpy()), "最大值:", np.max(t2_v8_0.asnumpy()))
    
    # 比较对应的输出结果
    print("\n== Max算子结果比较 ==")
    print("形状: T1:", t1_v3_0.shape, "T2:", t2_v8_0.shape)
    
    if np.array_equal(t1_v3_0.asnumpy(), t2_v8_0.asnumpy()):
        print("Max输出完全一致")
    else:
        mismatched = (t1_v3_0.asnumpy() != t2_v8_0.asnumpy())
        mismatched_count = np.sum(mismatched)
        total_elements = t1_v3_0.size
        mismatch_percentage = 100.0 * mismatched_count / total_elements
        print(f"不匹配元素数量: {mismatched_count} / {total_elements} ({mismatch_percentage:.1f}%)")
        
        # 计算差异
        abs_diff = np.abs(t1_v3_0.asnumpy() - t2_v8_0.asnumpy())
        max_diff = np.max(abs_diff)
        print(f"最大绝对差异: {max_diff}")
        
        # 找出差异最大的位置
        flat_idx = np.argmax(abs_diff.flatten())
        idx = np.unravel_index(flat_idx, abs_diff.shape)
        print(f"最大差异位置{idx}: T1={t1_v3_0.asnumpy()[idx]}, T2={t2_v8_0.asnumpy()[idx]}")
    
    print("\n== Less算子结果比较 ==")
    print("形状: T1:", t1_v4_0.shape, "T2:", t2_v6_0.shape)
    
    if np.array_equal(t1_v4_0.asnumpy(), t2_v6_0.asnumpy()):
        print("Less输出完全一致") 
    else:
        bool_mismatched = (t1_v4_0.asnumpy() != t2_v6_0.asnumpy())
        bool_mismatched_count = np.sum(bool_mismatched)
        bool_total = t1_v4_0.size
        bool_mismatch_percentage = 100.0 * bool_mismatched_count / bool_total
        print(f"不匹配元素数量: {bool_mismatched_count} / {bool_total} ({bool_mismatch_percentage:.1f}%)")
        
        # 显示部分不匹配的布尔值
        indices = np.where(bool_mismatched)
        for i in range(min(3, len(indices[0]))):
            idx = tuple(ind[i] for ind in indices)
            print(f"位置{idx}: T1={t1_v4_0.asnumpy()[idx]}, T2={t2_v6_0.asnumpy()[idx]}")

if __name__ == "__main__":
    reproduce_bug()

3 根因分析

******此处由用户填写******

4 解决方案

******此处由用户填写******

包含文字方案和最终脚本代码

请将正确的脚本打包并上传附件

我要发帖子