cann版本是8.2.RC1，python=3.11，torch=2.6， 设备 8卡 910B：
[rank2]: Traceback (most recent call last): [rank2]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/train.py", line 86, in <module> [rank2]: model.optimize_parameters() # calculate loss functions, get gradients, update network weights [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/cut_model.py", line 157, in optimize_parameters [rank2]: self.loss_G = self.compute_G_loss() [rank2]: ^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/cut_model.py", line 217, in compute_G_loss [rank2]: self.loss_NCE = self.calculate_NCE_loss(self.real_A, self.fake_B) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/cut_model.py", line 246, in calculate_NCE_loss [rank2]: feat_k_pool, sample_ids = self.netF(feat_k, self.opt.num_patches, None, src) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl [rank2]: return self._call_impl(*args, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl [rank2]: return forward_call(*args, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1643, in forward [rank2]: else self._run_ddp_forward(*inputs, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1459, in _run_ddp_forward [rank2]: return self.module(*inputs, **kwargs) # type: ignore[index] [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl [rank2]: return self._call_impl(*args, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl [rank2]: return forward_call(*args, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/networks.py", line 596, in forward [rank2]: patch_id = self._sample_foreground(feat, num_patches, real_A) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context [rank2]: return func(*args, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/networks.py", line 569, in _sample_foreground [rank2]: weights[background_mask] = 0.01 [rank2]: ~~~~~~~^^^^^^^^^^^^^^^^^ [rank2]: RuntimeError: The Inner error is reported as above. The process exits for this inner error, and the current working operator name is aclnnNonzeroV2. [rank2]: Since the operator is called asynchronously, the stacktrace may be inaccurate. If you want to get the accurate stacktrace, please set the environment variable ASCEND_LAUNCH_BLOCKING=1. [rank2]: Note: ASCEND_LAUNCH_BLOCKING=1 will force ops to run in synchronous mode, resulting in performance degradation. Please unset ASCEND_LAUNCH_BLOCKING in time after debugging. [rank2]: [ERROR] 2026-01-26-17:26:12 (PID:926003, Device:2, RankID:2) ERR00100 PTA call acl api failed. [rank5]:[E126 17:26:12.177434035 compiler_depend.ts:429] operator():build/CMakeFiles/torch_npu.dir/compiler_depend.ts:82 NPU function error: call aclnnNonzeroV2 failed, error code is 507048 [ERROR] 2026-01-26-17:26:12 (PID:926006, Device:5, RankID:5) ERR00100 PTA call acl api failed [Error]: The execution of the internal task times out. Rectify the fault based on the error information in the ascend log. EZ9999: Inner Error! rtStreamSynchronize execute failed, reason=[fftsplus timeout][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:53] EZ9999: [PID: 926006] 2026-01-26-17:26:12.491.491 rtStreamSynchronize failed. stream: 0x6a291390 TraceBack (most recent call last): Kernel Run failed. opType: 38, NonZero launch failed for NonZero, errno:507048. Exception raised from operator() at build/CMakeFiles/torch_npu.dir/compiler_depend.ts:82 (most recent call first): frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0xd4 (0x4002eeb7a9e4 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/lib/libc10.so) frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0xe4 (0x4002eeb2d384 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/lib/libc10.so) frame #2: <unknown function> + 0x1f43538 (0x4002fcf73538 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #3: <unknown function> + 0x20d9fc4 (0x4002fd109fc4 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #4: <unknown function> + 0x89fc70 (0x4002fb8cfc70 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #5: <unknown function> + 0x8a2004 (0x4002fb8d2004 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #6: <unknown function> + 0x89e92c (0x4002fb8ce92c in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #7: <unknown function> + 0xda294 (0x4002eecda294 in /home_data/home/songhy2024/miniconda3/envs/vae/bin/../lib/libstdc++.so.6) frame #8: <unknown function> + 0x878c (0x4002d63d878c in /usr/lib64/libpthread.so.0) frame #9: <unknown function> + 0xd508c (0x4002d661508c in /usr/lib64/libc.so.6) [rank5]: Traceback (most recent call last): [rank5]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/train.py", line 86, in <module> [rank5]: model.optimize_parameters() # calculate loss functions, get gradients, update network weights [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/cut_model.py", line 157, in optimize_parameters [rank5]: self.loss_G = self.compute_G_loss() [rank5]: ^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/cut_model.py", line 217, in compute_G_loss [rank5]: self.loss_NCE = self.calculate_NCE_loss(self.real_A, self.fake_B) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/cut_model.py", line 246, in calculate_NCE_loss [rank5]: feat_k_pool, sample_ids = self.netF(feat_k, self.opt.num_patches, None, src) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl [rank5]: return self._call_impl(*args, **kwargs) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl [rank5]: return forward_call(*args, **kwargs) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1643, in forward [rank5]: else self._run_ddp_forward(*inputs, **kwargs) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1459, in _run_ddp_forward [rank5]: return self.module(*inputs, **kwargs) # type: ignore[index] [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl [rank5]: return self._call_impl(*args, **kwargs) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl [rank5]: return forward_call(*args, **kwargs) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/networks.py", line 596, in forward [rank5]: patch_id = self._sample_foreground(feat, num_patches, real_A) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context [rank5]: return func(*args, **kwargs) [rank5]: ^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/networks.py", line 569, in _sample_foreground [rank5]: weights[background_mask] = 0.01 [rank5]: ~~~~~~~^^^^^^^^^^^^^^^^^ [rank5]: RuntimeError: The Inner error is reported as above. The process exits for this inner error, and the current working operator name is aclnnNonzeroV2. [rank5]: Since the operator is called asynchronously, the stacktrace may be inaccurate. If you want to get the accurate stacktrace, please set the environment variable ASCEND_LAUNCH_BLOCKING=1. [rank5]: Note: ASCEND_LAUNCH_BLOCKING=1 will force ops to run in synchronous mode, resulting in performance degradation. Please unset ASCEND_LAUNCH_BLOCKING in time after debugging. [rank5]: [ERROR] 2026-01-26-17:26:12 (PID:926006, Device:5, RankID:5) ERR00100 PTA call acl api failed. [rank6]:[E126 17:26:12.386059638 compiler_depend.ts:429] operator():build/CMakeFiles/torch_npu.dir/compiler_depend.ts:82 NPU function error: call aclnnNonzeroV2 failed, error code is 507048 [ERROR] 2026-01-26-17:26:12 (PID:926007, Device:6, RankID:6) ERR00100 PTA call acl api failed [Error]: The execution of the internal task times out. Rectify the fault based on the error information in the ascend log. EZ9999: Inner Error! rtStreamSynchronize execute failed, reason=[fftsplus timeout][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:53] EZ9999: [PID: 926007] 2026-01-26-17:26:12.700.012 rtStreamSynchronize failed. stream: 0x3fa21bc0 TraceBack (most recent call last): Kernel Run failed. opType: 38, NonZero launch failed for NonZero, errno:507048. Exception raised from operator() at build/CMakeFiles/torch_npu.dir/compiler_depend.ts:82 (most recent call first): frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0xd4 (0x40021c15a9e4 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/lib/libc10.so) frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0xe4 (0x40021c10d384 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/lib/libc10.so) frame #2: <unknown function> + 0x1f43538 (0x40022a553538 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #3: <unknown function> + 0x20d9fc4 (0x40022a6e9fc4 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #4: <unknown function> + 0x89fc70 (0x400228eafc70 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #5: <unknown function> + 0x8a2004 (0x400228eb2004 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #6: <unknown function> + 0x89e92c (0x400228eae92c in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #7: <unknown function> + 0xda294 (0x40021c2ba294 in /home_data/home/songhy2024/miniconda3/envs/vae/bin/../lib/libstdc++.so.6) frame #8: <unknown function> + 0x878c (0x4002039b878c in /usr/lib64/libpthread.so.0) frame #9: <unknown function> + 0xd508c (0x400203bf508c in /usr/lib64/libc.so.6)
code：
···
class PatchSampleF(nn.Module):
def __init__(self, use_mlp=False, init_type='normal', init_gain=0.02, nc=256, gpu_ids=[]):
# potential issues: currently, we use the same patch_ids for multiple images in the batch
super(PatchSampleF, self).__init__()
self.l2norm = Normalize(2)
self.use_mlp = use_mlp
self.nc = nc # hard-coded
self.mlp_init = False
self.init_type = init_type
self.init_gain = init_gain
self.gpu_ids = gpu_ids
def create_mlp(self, feats):
for mlp_id, feat in enumerate(feats):
input_nc = feat.shape[1]
mlp = nn.Sequential(*[nn.Linear(input_nc, self.nc), nn.ReLU(), nn.Linear(self.nc, self.nc)])
if len(self.gpu_ids) > 0:
mlp.cuda()
setattr(self, 'mlp_%d' % mlp_id, mlp)
init_net(self, self.init_type, self.init_gain, self.gpu_ids)
self.mlp_init = True
@torch.no_grad()
def _sample_foreground(self, feat, num_patches, real_A):
B, C, H, W = feat.shape
device = feat.device
total_pixels = H * W
mask = F.interpolate(real_A, size=(H, W), mode='nearest').clamp(-1, 1)
mask_bin = (mask.squeeze(1) > -0.999).float() # (B,H,W)
mask_flat = mask_bin.view(B, -1) # (B,H*W)
patch_ids_list = []
for b in range(B):
mb = mask_flat[b] # (H*W,)
weights = torch.ones(total_pixels, device=device) # 初始化为全1（前景权重）
background_mask = (mb <= 0.5) # 找出背景像素的索引
weights[background_mask] = 0.01
weights = weights / weights.sum()
idx = torch.multinomial(weights, min(num_patches, total_pixels), replacement=False)
patch_ids_list.append(idx)
return torch.cat(patch_ids_list)
def forward(self, feats, num_patches=64, patch_ids=None, real_A=None):
"""
feats : list of Tensor 多层特征
real_A : Tensor (B,1,H0,W0) 原始输入，用于生成前景mask
"""
return_ids = []
return_feats= []
if self.use_mlp and not self.mlp_init:
self.create_mlp(feats)
for feat_id, feat in enumerate(feats):
B, C, H, W = feat.shape
feat_reshape = feat.permute(0, 2, 3, 1).flatten(1, 2) # (B, H*W, C)
if num_patches > 0:
if patch_ids is not None:
patch_id = patch_ids[feat_id]
else:
# 关键：只在前景采样
patch_id = self._sample_foreground(feat, num_patches, real_A)
x_sample = feat_reshape[:, patch_id, :].flatten(0, 1) # (N, C)
else:
x_sample = feat_reshape
patch_id = []
if self.use_mlp:
mlp = getattr(self, 'mlp_%d' % feat_id)
x_sample = mlp(x_sample)
return_ids.append(patch_id)
x_sample = self.l2norm(x_sample)
if num_patches == 0:
x_sample = x_sample.permute(0, 2, 1).reshape([B, x_sample.shape[-1], H, W])
return_feats.append(x_sample)
return return_feats, return_ids
···
在训练到25epoch时会出现错误，
点赞
收藏
回复
分享
举报
已解决
cann版本是8.2.RC1，python=3.11，torch=2.6， 设备 8卡 910B：
[rank2]: Traceback (most recent call last): [rank2]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/train.py", line 86, in <module> [rank2]: model.optimize_parameters() # calculate loss functions, get gradients, update network weights [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/cut_model.py", line 157, in optimize_parameters [rank2]: self.loss_G = self.compute_G_loss() [rank2]: ^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/cut_model.py", line 217, in compute_G_loss [rank2]: self.loss_NCE = self.calculate_NCE_loss(self.real_A, self.fake_B) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/cut_model.py", line 246, in calculate_NCE_loss [rank2]: feat_k_pool, sample_ids = self.netF(feat_k, self.opt.num_patches, None, src) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl [rank2]: return self._call_impl(*args, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl [rank2]: return forward_call(*args, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1643, in forward [rank2]: else self._run_ddp_forward(*inputs, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1459, in _run_ddp_forward [rank2]: return self.module(*inputs, **kwargs) # type: ignore[index] [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl [rank2]: return self._call_impl(*args, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl [rank2]: return forward_call(*args, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/networks.py", line 596, in forward [rank2]: patch_id = self._sample_foreground(feat, num_patches, real_A) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context [rank2]: return func(*args, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/networks.py", line 569, in _sample_foreground [rank2]: weights[background_mask] = 0.01 [rank2]: ~~~~~~~^^^^^^^^^^^^^^^^^ [rank2]: RuntimeError: The Inner error is reported as above. The process exits for this inner error, and the current working operator name is aclnnNonzeroV2. [rank2]: Since the operator is called asynchronously, the stacktrace may be inaccurate. If you want to get the accurate stacktrace, please set the environment variable ASCEND_LAUNCH_BLOCKING=1. [rank2]: Note: ASCEND_LAUNCH_BLOCKING=1 will force ops to run in synchronous mode, resulting in performance degradation. Please unset ASCEND_LAUNCH_BLOCKING in time after debugging. [rank2]: [ERROR] 2026-01-26-17:26:12 (PID:926003, Device:2, RankID:2) ERR00100 PTA call acl api failed. [rank5]:[E126 17:26:12.177434035 compiler_depend.ts:429] operator():build/CMakeFiles/torch_npu.dir/compiler_depend.ts:82 NPU function error: call aclnnNonzeroV2 failed, error code is 507048 [ERROR] 2026-01-26-17:26:12 (PID:926006, Device:5, RankID:5) ERR00100 PTA call acl api failed [Error]: The execution of the internal task times out. Rectify the fault based on the error information in the ascend log. EZ9999: Inner Error! rtStreamSynchronize execute failed, reason=[fftsplus timeout][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:53] EZ9999: [PID: 926006] 2026-01-26-17:26:12.491.491 rtStreamSynchronize failed. stream: 0x6a291390 TraceBack (most recent call last): Kernel Run failed. opType: 38, NonZero launch failed for NonZero, errno:507048. Exception raised from operator() at build/CMakeFiles/torch_npu.dir/compiler_depend.ts:82 (most recent call first): frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0xd4 (0x4002eeb7a9e4 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/lib/libc10.so) frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0xe4 (0x4002eeb2d384 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/lib/libc10.so) frame #2: <unknown function> + 0x1f43538 (0x4002fcf73538 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #3: <unknown function> + 0x20d9fc4 (0x4002fd109fc4 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #4: <unknown function> + 0x89fc70 (0x4002fb8cfc70 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #5: <unknown function> + 0x8a2004 (0x4002fb8d2004 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #6: <unknown function> + 0x89e92c (0x4002fb8ce92c in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #7: <unknown function> + 0xda294 (0x4002eecda294 in /home_data/home/songhy2024/miniconda3/envs/vae/bin/../lib/libstdc++.so.6) frame #8: <unknown function> + 0x878c (0x4002d63d878c in /usr/lib64/libpthread.so.0) frame #9: <unknown function> + 0xd508c (0x4002d661508c in /usr/lib64/libc.so.6) [rank5]: Traceback (most recent call last): [rank5]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/train.py", line 86, in <module> [rank5]: model.optimize_parameters() # calculate loss functions, get gradients, update network weights [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/cut_model.py", line 157, in optimize_parameters [rank5]: self.loss_G = self.compute_G_loss() [rank5]: ^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/cut_model.py", line 217, in compute_G_loss [rank5]: self.loss_NCE = self.calculate_NCE_loss(self.real_A, self.fake_B) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/cut_model.py", line 246, in calculate_NCE_loss [rank5]: feat_k_pool, sample_ids = self.netF(feat_k, self.opt.num_patches, None, src) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl [rank5]: return self._call_impl(*args, **kwargs) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl [rank5]: return forward_call(*args, **kwargs) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1643, in forward [rank5]: else self._run_ddp_forward(*inputs, **kwargs) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1459, in _run_ddp_forward [rank5]: return self.module(*inputs, **kwargs) # type: ignore[index] [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl [rank5]: return self._call_impl(*args, **kwargs) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl [rank5]: return forward_call(*args, **kwargs) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/networks.py", line 596, in forward [rank5]: patch_id = self._sample_foreground(feat, num_patches, real_A) [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context [rank5]: return func(*args, **kwargs) [rank5]: ^^^^^^^^^^^^^^^^^^^^^ [rank5]: File "/public/home_data/home/songhy2024/contrastive-unpaired-translation/models/networks.py", line 569, in _sample_foreground [rank5]: weights[background_mask] = 0.01 [rank5]: ~~~~~~~^^^^^^^^^^^^^^^^^ [rank5]: RuntimeError: The Inner error is reported as above. The process exits for this inner error, and the current working operator name is aclnnNonzeroV2. [rank5]: Since the operator is called asynchronously, the stacktrace may be inaccurate. If you want to get the accurate stacktrace, please set the environment variable ASCEND_LAUNCH_BLOCKING=1. [rank5]: Note: ASCEND_LAUNCH_BLOCKING=1 will force ops to run in synchronous mode, resulting in performance degradation. Please unset ASCEND_LAUNCH_BLOCKING in time after debugging. [rank5]: [ERROR] 2026-01-26-17:26:12 (PID:926006, Device:5, RankID:5) ERR00100 PTA call acl api failed. [rank6]:[E126 17:26:12.386059638 compiler_depend.ts:429] operator():build/CMakeFiles/torch_npu.dir/compiler_depend.ts:82 NPU function error: call aclnnNonzeroV2 failed, error code is 507048 [ERROR] 2026-01-26-17:26:12 (PID:926007, Device:6, RankID:6) ERR00100 PTA call acl api failed [Error]: The execution of the internal task times out. Rectify the fault based on the error information in the ascend log. EZ9999: Inner Error! rtStreamSynchronize execute failed, reason=[fftsplus timeout][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:53] EZ9999: [PID: 926007] 2026-01-26-17:26:12.700.012 rtStreamSynchronize failed. stream: 0x3fa21bc0 TraceBack (most recent call last): Kernel Run failed. opType: 38, NonZero launch failed for NonZero, errno:507048. Exception raised from operator() at build/CMakeFiles/torch_npu.dir/compiler_depend.ts:82 (most recent call first): frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0xd4 (0x40021c15a9e4 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/lib/libc10.so) frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0xe4 (0x40021c10d384 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch/lib/libc10.so) frame #2: <unknown function> + 0x1f43538 (0x40022a553538 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #3: <unknown function> + 0x20d9fc4 (0x40022a6e9fc4 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #4: <unknown function> + 0x89fc70 (0x400228eafc70 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #5: <unknown function> + 0x8a2004 (0x400228eb2004 in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #6: <unknown function> + 0x89e92c (0x400228eae92c in /home_data/home/songhy2024/miniconda3/envs/vae/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #7: <unknown function> + 0xda294 (0x40021c2ba294 in /home_data/home/songhy2024/miniconda3/envs/vae/bin/../lib/libstdc++.so.6) frame #8: <unknown function> + 0x878c (0x4002039b878c in /usr/lib64/libpthread.so.0) frame #9: <unknown function> + 0xd508c (0x400203bf508c in /usr/lib64/libc.so.6)
code：
···
class PatchSampleF(nn.Module):
def __init__(self, use_mlp=False, init_type='normal', init_gain=0.02, nc=256, gpu_ids=[]):
# potential issues: currently, we use the same patch_ids for multiple images in the batch
super(PatchSampleF, self).__init__()
self.l2norm = Normalize(2)
self.use_mlp = use_mlp
self.nc = nc # hard-coded
self.mlp_init = False
self.init_type = init_type
self.init_gain = init_gain
self.gpu_ids = gpu_ids
def create_mlp(self, feats):
for mlp_id, feat in enumerate(feats):
input_nc = feat.shape[1]
mlp = nn.Sequential(*[nn.Linear(input_nc, self.nc), nn.ReLU(), nn.Linear(self.nc, self.nc)])
if len(self.gpu_ids) > 0:
mlp.cuda()
setattr(self, 'mlp_%d' % mlp_id, mlp)
init_net(self, self.init_type, self.init_gain, self.gpu_ids)
self.mlp_init = True
@torch.no_grad()
def _sample_foreground(self, feat, num_patches, real_A):
B, C, H, W = feat.shape
device = feat.device
total_pixels = H * W
mask = F.interpolate(real_A, size=(H, W), mode='nearest').clamp(-1, 1)
mask_bin = (mask.squeeze(1) > -0.999).float() # (B,H,W)
mask_flat = mask_bin.view(B, -1) # (B,H*W)
patch_ids_list = []
for b in range(B):
mb = mask_flat[b] # (H*W,)
weights = torch.ones(total_pixels, device=device) # 初始化为全1（前景权重）
background_mask = (mb <= 0.5) # 找出背景像素的索引
weights[background_mask] = 0.01
weights = weights / weights.sum()
idx = torch.multinomial(weights, min(num_patches, total_pixels), replacement=False)
patch_ids_list.append(idx)
return torch.cat(patch_ids_list)
def forward(self, feats, num_patches=64, patch_ids=None, real_A=None):
"""
feats : list of Tensor 多层特征
real_A : Tensor (B,1,H0,W0) 原始输入，用于生成前景mask
"""
return_ids = []
return_feats= []
if self.use_mlp and not self.mlp_init:
self.create_mlp(feats)
for feat_id, feat in enumerate(feats):
B, C, H, W = feat.shape
feat_reshape = feat.permute(0, 2, 3, 1).flatten(1, 2) # (B, H*W, C)
if num_patches > 0:
if patch_ids is not None:
patch_id = patch_ids[feat_id]
else:
# 关键：只在前景采样
patch_id = self._sample_foreground(feat, num_patches, real_A)
x_sample = feat_reshape[:, patch_id, :].flatten(0, 1) # (N, C)
else:
x_sample = feat_reshape
patch_id = []
if self.use_mlp:
mlp = getattr(self, 'mlp_%d' % feat_id)
x_sample = mlp(x_sample)
return_ids.append(patch_id)
x_sample = self.l2norm(x_sample)
if num_patches == 0:
x_sample = x_sample.permute(0, 2, 1).reshape([B, x_sample.shape[-1], H, W])
return_feats.append(x_sample)
return return_feats, return_ids
···
在训练到25epoch时会出现错误，