cann版本是8.3.RC1.alpha001,torch_npu版本是2.8.0rc1
报错内容:
Training: 31%|██▊ | 1837/6000 [1:10:28<2:31:16, 2.18s/it, loss=0.018, grad=0.93, lr=9.12e-05]
Training: 31%|██▊ | 1837/6000 [1:10:30<2:31:16, 2.18s/it, loss=0.028, grad=0.67, lr=9.12e-05]
Training: 31%|██▊ | 1838/6000 [1:10:30<2:32:09, 2.19s/it, loss=0.028, grad=0.67, lr=9.12e-05][rank1]: Traceback (most recent call last):
[rank1]: File "/root/anaconda3/envs/lerobot/lib/python3.10/runpy.py", line 196, in _run_module_as_main
[rank1]: return _run_code(code, main_globals, None,
[rank1]: File "/root/anaconda3/envs/lerobot/lib/python3.10/runpy.py", line 86, in _run_code
[rank1]: exec(code, run_globals)
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/scripts/lerobot_train.py", line 448, in <module>
[rank1]: main()
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/scripts/lerobot_train.py", line 444, in main
[rank1]: train()
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/configs/parser.py", line 225, in wrapper_inner
[rank1]: response = fn(cfg, *args, **kwargs)
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/scripts/lerobot_train.py", line 355, in train
[rank1]: train_tracker, output_dict = update_policy(
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/scripts/lerobot_train.py", line 158, in update_policy
[rank1]: loss, output_dict = policy.forward(batch)
[rank1]: File "/root/anaconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1648, in forward
[rank1]: else self._run_ddp_forward(*inputs, **kwargs)
[rank1]: File "/root/anaconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1474, in _run_ddp_forward
[rank1]: return self.module(*inputs, **kwargs) # type: ignore[index]
[rank1]: File "/root/anaconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: File "/root/anaconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
[rank1]: return forward_call(*args, **kwargs)
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/policies/smolvla/modeling_smolvla.py", line 326, in forward
[rank1]: losses = self.model.forward(images, img_masks, lang_tokens, lang_masks, state, actions, noise, time)
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/policies/smolvla/modeling_smolvla.py", line 678, in forward
[rank1]: time = self.sample_time(actions.shape[0], actions.device)
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/policies/smolvla/modeling_smolvla.py", line 529, in sample_time
[rank1]: time_beta = beta_dist.sample((bsize,)).to(device=device, dtype=torch.float32)
[rank1]: RuntimeError: ACL stream synchronize failed, error code:107020
cann版本是8.3.RC1.alpha001,torch_npu版本是2.8.0rc1
报错内容:
Training: 31%|██▊ | 1837/6000 [1:10:28<2:31:16, 2.18s/it, loss=0.018, grad=0.93, lr=9.12e-05]
Training: 31%|██▊ | 1837/6000 [1:10:30<2:31:16, 2.18s/it, loss=0.028, grad=0.67, lr=9.12e-05]
Training: 31%|██▊ | 1838/6000 [1:10:30<2:32:09, 2.19s/it, loss=0.028, grad=0.67, lr=9.12e-05][rank1]: Traceback (most recent call last):
[rank1]: File "/root/anaconda3/envs/lerobot/lib/python3.10/runpy.py", line 196, in _run_module_as_main
[rank1]: return _run_code(code, main_globals, None,
[rank1]: File "/root/anaconda3/envs/lerobot/lib/python3.10/runpy.py", line 86, in _run_code
[rank1]: exec(code, run_globals)
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/scripts/lerobot_train.py", line 448, in <module>
[rank1]: main()
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/scripts/lerobot_train.py", line 444, in main
[rank1]: train()
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/configs/parser.py", line 225, in wrapper_inner
[rank1]: response = fn(cfg, *args, **kwargs)
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/scripts/lerobot_train.py", line 355, in train
[rank1]: train_tracker, output_dict = update_policy(
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/scripts/lerobot_train.py", line 158, in update_policy
[rank1]: loss, output_dict = policy.forward(batch)
[rank1]: File "/root/anaconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1648, in forward
[rank1]: else self._run_ddp_forward(*inputs, **kwargs)
[rank1]: File "/root/anaconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1474, in _run_ddp_forward
[rank1]: return self.module(*inputs, **kwargs) # type: ignore[index]
[rank1]: File "/root/anaconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: File "/root/anaconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
[rank1]: return forward_call(*args, **kwargs)
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/policies/smolvla/modeling_smolvla.py", line 326, in forward
[rank1]: losses = self.model.forward(images, img_masks, lang_tokens, lang_masks, state, actions, noise, time)
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/policies/smolvla/modeling_smolvla.py", line 678, in forward
[rank1]: time = self.sample_time(actions.shape[0], actions.device)
[rank1]: File "/root/zzx/workspace/lerobot_ascend/src/lerobot/policies/smolvla/modeling_smolvla.py", line 529, in sample_time
[rank1]: time_beta = beta_dist.sample((bsize,)).to(device=device, dtype=torch.float32)
[rank1]: RuntimeError: ACL stream synchronize failed, error code:107020