Traceback (most recent call last):
File "/home/xiaoyu/anaconda3/envs/LTVR/lib/python3.9/runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/xiaoyu/anaconda3/envs/LTVR/lib/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/home/xiaoyu/.vscode-server/extensions/ms-python.python-2022.18.2/pythonFiles/lib/python/debugpy/adapter/../../debugpy/launcher/../../debugpy/__main__.py", line 39, in <module>
cli.main()
File "/home/xiaoyu/.vscode-server/extensions/ms-python.python-2022.18.2/pythonFiles/lib/python/debugpy/adapter/../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py", line 430, in main
run()
File "/home/xiaoyu/.vscode-server/extensions/ms-python.python-2022.18.2/pythonFiles/lib/python/debugpy/adapter/../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py", line 284, in run_file
runpy.run_path(target, run_name="__main__")
File "/home/xiaoyu/.vscode-server/extensions/ms-python.python-2022.18.2/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 321, in run_path
return _run_module_code(code, init_globals, run_name,
File "/home/xiaoyu/.vscode-server/extensions/ms-python.python-2022.18.2/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 135, in _run_module_code
_run_code(code, mod_globals, init_globals,
File "/home/xiaoyu/.vscode-server/extensions/ms-python.python-2022.18.2/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 124, in _run_code
exec(code, run_globals)
File "/home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/main.py", line 244, in <module>
main()
File "/home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/main.py", line 33, in main
main_worker(args)
File "/home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/main.py", line 88, in main_worker
train(train_dataset_weak, model, optimizer, epoch, args, log_training, tf_writer)
File "/home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/main.py", line 157, in train
train_step(image, label)
File "/home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/main.py", line 144, in train_step
loss, out, loss2 = forward_fn(image, label)
File "/home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/main.py", line 132, in forward_fn
z2 = model(image,idx=0)
File "/home/xiaoyu/anaconda3/envs/LTVR/lib/python3.9/site-packages/mindspore/nn/cell.py", line 596, in __call__
out = self.compile_and_run(*args)
File "/home/xiaoyu/anaconda3/envs/LTVR/lib/python3.9/site-packages/mindspore/nn/cell.py", line 985, in compile_and_run
self.compile(*inputs)
File "/home/xiaoyu/anaconda3/envs/LTVR/lib/python3.9/site-packages/mindspore/nn/cell.py", line 956, in compile
_cell_graph_executor.compile(self, *inputs, phase=self.phase, auto_parallel_mode=self._auto_parallel_mode,
File "/home/xiaoyu/anaconda3/envs/LTVR/lib/python3.9/site-packages/mindspore/common/api.py", line 1131, in compile
result = self._graph_executor.compile(obj, args_list, phase, self._use_vm_mode())
RuntimeError: The 'mul' operation does not support the type [Anything, Tensor[Float32]].
The supported types of overload function `mul` is: [Tensor, COOTensor], [Tensor, CSRTensor], [CSRTensor, Tensor], [List, Tensor], [Tensor, List], [Tuple, Tensor], [Number, String], [Number, Tensor], [COOTensor, Tensor], [Tensor, Tensor], [String, Number], [Number, List], [Tensor, Number], [Tensor, Tuple], [Number, Number], [Tuple, Number], [List, Number], [Number, Tuple].
----------------------------------------------------
- The Traceback of Net Construct Code:
----------------------------------------------------
The function call stack (See file '/home/xiaoyu/rank_0/om/analyze_fail.dat' for more details. Get instructions about `analyze_fail.dat` at https://www.mindspore.cn/search?inputValue=analyze_fail.dat):
# 0 In file /home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/models/resnet_synthetic.py:138
out, _ = self.bn1(out,idx)
^
# 1 In file /home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/models/gmm_resnet.py:239
if split_flag == 0:
# 2 In file /home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/models/gmm_resnet.py:240
return self.split_bn(input),split_flag
^
# 3 In file /home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/models/gmm_resnet.py:217
for idx in range(self.num_mixtures):
^
# 4 In file /home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/models/gmm_resnet.py:224
if self.training: # aux BN only relevant while training
# 5 In file /home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/models/gmm_resnet.py:226
self.running_mean[idx] = momentum * mean + (1-momentum) * self.running_mean[idx]
Pynative模型训练正常,但在Graph模式下报错RuntimeError: The 'mul' operation does not support the type [Anything, Tensor[Float32]].
Traceback (most recent call last): File "/home/xiaoyu/anaconda3/envs/LTVR/lib/python3.9/runpy.py", line 197, in _run_module_as_main return _run_code(code, main_globals, None, File "/home/xiaoyu/anaconda3/envs/LTVR/lib/python3.9/runpy.py", line 87, in _run_code exec(code, run_globals) File "/home/xiaoyu/.vscode-server/extensions/ms-python.python-2022.18.2/pythonFiles/lib/python/debugpy/adapter/../../debugpy/launcher/../../debugpy/__main__.py", line 39, in <module> cli.main() File "/home/xiaoyu/.vscode-server/extensions/ms-python.python-2022.18.2/pythonFiles/lib/python/debugpy/adapter/../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py", line 430, in main run() File "/home/xiaoyu/.vscode-server/extensions/ms-python.python-2022.18.2/pythonFiles/lib/python/debugpy/adapter/../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py", line 284, in run_file runpy.run_path(target, run_name="__main__") File "/home/xiaoyu/.vscode-server/extensions/ms-python.python-2022.18.2/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 321, in run_path return _run_module_code(code, init_globals, run_name, File "/home/xiaoyu/.vscode-server/extensions/ms-python.python-2022.18.2/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 135, in _run_module_code _run_code(code, mod_globals, init_globals, File "/home/xiaoyu/.vscode-server/extensions/ms-python.python-2022.18.2/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 124, in _run_code exec(code, run_globals) File "/home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/main.py", line 244, in <module> main() File "/home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/main.py", line 33, in main main_worker(args) File "/home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/main.py", line 88, in main_worker train(train_dataset_weak, model, optimizer, epoch, args, log_training, tf_writer) File "/home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/main.py", line 157, in train train_step(image, label) File "/home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/main.py", line 144, in train_step loss, out, loss2 = forward_fn(image, label) File "/home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/main.py", line 132, in forward_fn z2 = model(image,idx=0) File "/home/xiaoyu/anaconda3/envs/LTVR/lib/python3.9/site-packages/mindspore/nn/cell.py", line 596, in __call__ out = self.compile_and_run(*args) File "/home/xiaoyu/anaconda3/envs/LTVR/lib/python3.9/site-packages/mindspore/nn/cell.py", line 985, in compile_and_run self.compile(*inputs) File "/home/xiaoyu/anaconda3/envs/LTVR/lib/python3.9/site-packages/mindspore/nn/cell.py", line 956, in compile _cell_graph_executor.compile(self, *inputs, phase=self.phase, auto_parallel_mode=self._auto_parallel_mode, File "/home/xiaoyu/anaconda3/envs/LTVR/lib/python3.9/site-packages/mindspore/common/api.py", line 1131, in compile result = self._graph_executor.compile(obj, args_list, phase, self._use_vm_mode()) RuntimeError: The 'mul' operation does not support the type [Anything, Tensor[Float32]]. The supported types of overload function `mul` is: [Tensor, COOTensor], [Tensor, CSRTensor], [CSRTensor, Tensor], [List, Tensor], [Tensor, List], [Tuple, Tensor], [Number, String], [Number, Tensor], [COOTensor, Tensor], [Tensor, Tensor], [String, Number], [Number, List], [Tensor, Number], [Tensor, Tuple], [Number, Number], [Tuple, Number], [List, Number], [Number, Tuple]. ---------------------------------------------------- - The Traceback of Net Construct Code: ---------------------------------------------------- The function call stack (See file '/home/xiaoyu/rank_0/om/analyze_fail.dat' for more details. Get instructions about `analyze_fail.dat` at https://www.mindspore.cn/search?inputValue=analyze_fail.dat): # 0 In file /home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/models/resnet_synthetic.py:138 out, _ = self.bn1(out,idx) ^ # 1 In file /home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/models/gmm_resnet.py:239 if split_flag == 0: # 2 In file /home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/models/gmm_resnet.py:240 return self.split_bn(input),split_flag ^ # 3 In file /home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/models/gmm_resnet.py:217 for idx in range(self.num_mixtures): ^ # 4 In file /home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/models/gmm_resnet.py:224 if self.training: # aux BN only relevant while training # 5 In file /home/xiaoyu/workspace/LTVR/LTVR_mindspore_graph/models/gmm_resnet.py:226 self.running_mean[idx] = momentum * mean + (1-momentum) * self.running_mean[idx]