import numpy as np
import mindspore as ms
import mindspore.nn as nn
import mindspore.ops as ops
class T1Model(nn.Cell):
def __init__(self):
super(T1Model, self).__init__()
self.cast = ops.Cast()
def construct(self, v5_0, v6_0, v7_0):
# v5_0: b[5, 14, 1, 14, 57], v6_0: u8[57], v7_0: u8[1, 1, 57]
v8_0 = ops.where(v5_0, v6_0, v7_0) # core.Where
v3_0 = ops.maximum(v8_0, v8_0) # core.Max
v1_0 = self.cast(v8_0, ms.int8) # Cast to int8
v2_0 = ops.minimum(v1_0, v1_0) # core.Min
v4_0 = ops.less(v2_0, v1_0) # core.Less
# 返回多个结果方便分析
return v3_0, v4_0
class T2Model(nn.Cell):
def __init__(self):
super(T2Model, self).__init__()
self.cast = ops.Cast()
def construct(self, v2_0, v1_0, v0_0):
# v2_0: b[5, 14, 1, 14, 57], v1_0: u8[57], v0_0: u8[1, 1, 57]
v3_0 = ops.where(v2_0, v1_0, v0_0) # core.Where
v4_0 = self.cast(v3_0, ms.int8) # Cast to int8
v5_0 = ops.minimum(v4_0, v4_0) # core.Min
v6_0 = ops.less(v5_0, v4_0) # core.Less
v8_0 = ops.maximum(v3_0, v3_0) # core.Max
# 返回多个结果方便分析
return v8_0, v6_0
def reproduce_bug():
# 设置环境
ms.set_context(mode=ms.PYNATIVE_MODE)
ms.set_device("GPU")
# 创建模型
t1 = T1Model()
t2 = T2Model()
# 根据图创建输入 - 严格按照DOT文件中的形状和类型
np.random.seed(42)
shape1 = (5, 14, 1, 14, 57)
shape2 = (57,)
shape3 = (1, 1, 57)
# T1输入
v5_0_t1 = ms.Tensor(np.random.choice([True, False], size=shape1))
v6_0_t1 = ms.Tensor(np.random.randint(0, 255, shape2, dtype=np.uint8))
v7_0_t1 = ms.Tensor(np.random.randint(0, 255, shape3, dtype=np.uint8))
# T2输入 - 保持相同输入以便比较
v2_0_t2 = v5_0_t1 # 布尔掩码
v1_0_t2 = v6_0_t1 # uint8[57]
v0_0_t2 = v7_0_t1 # uint8[1, 1, 57]
# 执行模型
t1_v3_0, t1_v4_0 = t1(v5_0_t1, v6_0_t1, v7_0_t1)
t2_v8_0, t2_v6_0 = t2(v2_0_t2, v1_0_t2, v0_0_t2)
# 打印T1和T2的输出
print("\n== T1模型输出 ==")
print("Max结果 - 形状:", t1_v3_0.shape, "数据类型:", t1_v3_0.dtype)
print("部分值 (前5个元素):", t1_v3_0.asnumpy().flatten()[:5])
print("统计信息 - 最小值:", np.min(t1_v3_0.asnumpy()), "最大值:", np.max(t1_v3_0.asnumpy()))
print("\n== T2模型输出 ==")
print("Max结果 - 形状:", t2_v8_0.shape, "数据类型:", t2_v8_0.dtype)
print("部分值 (前5个元素):", t2_v8_0.asnumpy().flatten()[:5])
print("统计信息 - 最小值:", np.min(t2_v8_0.asnumpy()), "最大值:", np.max(t2_v8_0.asnumpy()))
# 比较对应的输出结果
print("\n== Max算子结果比较 ==")
print("形状: T1:", t1_v3_0.shape, "T2:", t2_v8_0.shape)
if np.array_equal(t1_v3_0.asnumpy(), t2_v8_0.asnumpy()):
print("Max输出完全一致")
else:
mismatched = (t1_v3_0.asnumpy() != t2_v8_0.asnumpy())
mismatched_count = np.sum(mismatched)
total_elements = t1_v3_0.size
mismatch_percentage = 100.0 * mismatched_count / total_elements
print(f"不匹配元素数量: {mismatched_count} / {total_elements} ({mismatch_percentage:.1f}%)")
# 计算差异
abs_diff = np.abs(t1_v3_0.asnumpy() - t2_v8_0.asnumpy())
max_diff = np.max(abs_diff)
print(f"最大绝对差异: {max_diff}")
# 找出差异最大的位置
flat_idx = np.argmax(abs_diff.flatten())
idx = np.unravel_index(flat_idx, abs_diff.shape)
print(f"最大差异位置{idx}: T1={t1_v3_0.asnumpy()[idx]}, T2={t2_v8_0.asnumpy()[idx]}")
print("\n== Less算子结果比较 ==")
print("形状: T1:", t1_v4_0.shape, "T2:", t2_v6_0.shape)
if np.array_equal(t1_v4_0.asnumpy(), t2_v6_0.asnumpy()):
print("Less输出完全一致")
else:
bool_mismatched = (t1_v4_0.asnumpy() != t2_v6_0.asnumpy())
bool_mismatched_count = np.sum(bool_mismatched)
bool_total = t1_v4_0.size
bool_mismatch_percentage = 100.0 * bool_mismatched_count / bool_total
print(f"不匹配元素数量: {bool_mismatched_count} / {bool_total} ({bool_mismatch_percentage:.1f}%)")
# 显示部分不匹配的布尔值
indices = np.where(bool_mismatched)
for i in range(min(3, len(indices[0]))):
idx = tuple(ind[i] for ind in indices)
print(f"位置{idx}: T1={t1_v4_0.asnumpy()[idx]}, T2={t2_v6_0.asnumpy()[idx]}")
if __name__ == "__main__":
reproduce_bug()
1 系统环境
硬件环境(Ascend/GPU/CPU): GPU
MindSpore版本: mindspore=2.5.0
执行模式(PyNative/ Graph): PyNative
Python版本: Python=3.10
操作系统平台: Ubuntu22.04
2 报错信息
2.1 问题描述
在MindSpore GPU模式下,两个数学上等价的模型(涉及uint8→int8的Cast、Where、Max等操作)推理结果出现不一致。理论上两个模型输出应完全一致,但实际Max结果有0.2%的元素不匹配,最大差异高达251。而Less操作输出完全一致,表明问题出现在Max操作在Cast前后的位置差异导致的整型数据流异常。
定义两个数学上等价的模型:
2.2 推理脚本
2.3 结果信息
2.4 脚本信息
理论上两个模型输出应完全一致,但实际Max结果有0.2%的元素不匹配,最大差异高达251。而Less结果完全一致,表明问题出现在Max操作的位置不同导致的整型数据流异常。
import numpy as np import mindspore as ms import mindspore.nn as nn import mindspore.ops as ops class T1Model(nn.Cell): def __init__(self): super(T1Model, self).__init__() self.cast = ops.Cast() def construct(self, v5_0, v6_0, v7_0): # v5_0: b[5, 14, 1, 14, 57], v6_0: u8[57], v7_0: u8[1, 1, 57] v8_0 = ops.where(v5_0, v6_0, v7_0) # core.Where v3_0 = ops.maximum(v8_0, v8_0) # core.Max v1_0 = self.cast(v8_0, ms.int8) # Cast to int8 v2_0 = ops.minimum(v1_0, v1_0) # core.Min v4_0 = ops.less(v2_0, v1_0) # core.Less # 返回多个结果方便分析 return v3_0, v4_0 class T2Model(nn.Cell): def __init__(self): super(T2Model, self).__init__() self.cast = ops.Cast() def construct(self, v2_0, v1_0, v0_0): # v2_0: b[5, 14, 1, 14, 57], v1_0: u8[57], v0_0: u8[1, 1, 57] v3_0 = ops.where(v2_0, v1_0, v0_0) # core.Where v4_0 = self.cast(v3_0, ms.int8) # Cast to int8 v5_0 = ops.minimum(v4_0, v4_0) # core.Min v6_0 = ops.less(v5_0, v4_0) # core.Less v8_0 = ops.maximum(v3_0, v3_0) # core.Max # 返回多个结果方便分析 return v8_0, v6_0 def reproduce_bug(): # 设置环境 ms.set_context(mode=ms.PYNATIVE_MODE) ms.set_device("GPU") # 创建模型 t1 = T1Model() t2 = T2Model() # 根据图创建输入 - 严格按照DOT文件中的形状和类型 np.random.seed(42) shape1 = (5, 14, 1, 14, 57) shape2 = (57,) shape3 = (1, 1, 57) # T1输入 v5_0_t1 = ms.Tensor(np.random.choice([True, False], size=shape1)) v6_0_t1 = ms.Tensor(np.random.randint(0, 255, shape2, dtype=np.uint8)) v7_0_t1 = ms.Tensor(np.random.randint(0, 255, shape3, dtype=np.uint8)) # T2输入 - 保持相同输入以便比较 v2_0_t2 = v5_0_t1 # 布尔掩码 v1_0_t2 = v6_0_t1 # uint8[57] v0_0_t2 = v7_0_t1 # uint8[1, 1, 57] # 执行模型 t1_v3_0, t1_v4_0 = t1(v5_0_t1, v6_0_t1, v7_0_t1) t2_v8_0, t2_v6_0 = t2(v2_0_t2, v1_0_t2, v0_0_t2) # 打印T1和T2的输出 print("\n== T1模型输出 ==") print("Max结果 - 形状:", t1_v3_0.shape, "数据类型:", t1_v3_0.dtype) print("部分值 (前5个元素):", t1_v3_0.asnumpy().flatten()[:5]) print("统计信息 - 最小值:", np.min(t1_v3_0.asnumpy()), "最大值:", np.max(t1_v3_0.asnumpy())) print("\n== T2模型输出 ==") print("Max结果 - 形状:", t2_v8_0.shape, "数据类型:", t2_v8_0.dtype) print("部分值 (前5个元素):", t2_v8_0.asnumpy().flatten()[:5]) print("统计信息 - 最小值:", np.min(t2_v8_0.asnumpy()), "最大值:", np.max(t2_v8_0.asnumpy())) # 比较对应的输出结果 print("\n== Max算子结果比较 ==") print("形状: T1:", t1_v3_0.shape, "T2:", t2_v8_0.shape) if np.array_equal(t1_v3_0.asnumpy(), t2_v8_0.asnumpy()): print("Max输出完全一致") else: mismatched = (t1_v3_0.asnumpy() != t2_v8_0.asnumpy()) mismatched_count = np.sum(mismatched) total_elements = t1_v3_0.size mismatch_percentage = 100.0 * mismatched_count / total_elements print(f"不匹配元素数量: {mismatched_count} / {total_elements} ({mismatch_percentage:.1f}%)") # 计算差异 abs_diff = np.abs(t1_v3_0.asnumpy() - t2_v8_0.asnumpy()) max_diff = np.max(abs_diff) print(f"最大绝对差异: {max_diff}") # 找出差异最大的位置 flat_idx = np.argmax(abs_diff.flatten()) idx = np.unravel_index(flat_idx, abs_diff.shape) print(f"最大差异位置{idx}: T1={t1_v3_0.asnumpy()[idx]}, T2={t2_v8_0.asnumpy()[idx]}") print("\n== Less算子结果比较 ==") print("形状: T1:", t1_v4_0.shape, "T2:", t2_v6_0.shape) if np.array_equal(t1_v4_0.asnumpy(), t2_v6_0.asnumpy()): print("Less输出完全一致") else: bool_mismatched = (t1_v4_0.asnumpy() != t2_v6_0.asnumpy()) bool_mismatched_count = np.sum(bool_mismatched) bool_total = t1_v4_0.size bool_mismatch_percentage = 100.0 * bool_mismatched_count / bool_total print(f"不匹配元素数量: {bool_mismatched_count} / {bool_total} ({bool_mismatch_percentage:.1f}%)") # 显示部分不匹配的布尔值 indices = np.where(bool_mismatched) for i in range(min(3, len(indices[0]))): idx = tuple(ind[i] for ind in indices) print(f"位置{idx}: T1={t1_v4_0.asnumpy()[idx]}, T2={t2_v6_0.asnumpy()[idx]}") if __name__ == "__main__": reproduce_bug()3 根因分析
******此处由用户填写******
4 解决方案
******此处由用户填写******
包含文字方案和最终脚本代码
请将正确的脚本打包并上传附件