[rank15]: Traceback (most recent call last):
[rank15]: File "/opt/tiger/ms-swift/swift/cli/_megatron/sft.py", line 7, in <module>
[rank15]: megatron_sft_main()
[rank15]: File "/opt/tiger/ms-swift/swift/megatron/pipelines/train/sft.py", line 97, in megatron_sft_main
[rank15]: return MegatronSft(args).main()
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/tiger/ms-swift/swift/pipelines/base.py", line 52, in main
[rank15]: result = self.run()
[rank15]: ^^^^^^^^^^
[rank15]: File "/opt/tiger/ms-swift/swift/megatron/pipelines/train/sft.py", line 72, in run
[rank15]: trainer.train(train_dataset, val_dataset)
[rank15]: File "/opt/tiger/ms-swift/swift/megatron/trainers/base.py", line 636, in train
[rank15]: metrics, grad_norm, update_successful = self.train_step(train_data_iterator)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/tiger/ms-swift/swift/megatron/trainers/base.py", line 857, in train_step
[rank15]: metrics = forward_backward_func(
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/pipeline_parallel/schedules.py", line 656, in forward_backward_no_pipelining
[rank15]: output_tensor, num_tokens = forward_step(
[rank15]: ^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/pipeline_parallel/schedules.py", line 423, in forward_step
[rank15]: output_tensor, loss_func = forward_step_func(data_iterator, model)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/tiger/ms-swift/swift/megatron/trainers/trainer.py", line 124, in forward_step
[rank15]: output_tensor = model(**data)
[rank15]: ^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/distributed/data_parallel_base.py", line 22, in forward
[rank15]: return self.module(*inputs, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/module.py", line 489, in forward
[rank15]: outputs = self.module(*inputs, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/mm_gpt_model.py", line 95, in forward
[rank15]: return self.language_model(
[rank15]: ^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/gpt_model.py", line 310, in forward
[rank15]: decoder_output = self.decoder(
[rank15]: ^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/transformer_block.py", line 619, in __call__
[rank15]: return super().__call__(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/module.py", line 352, in __call__
[rank15]: return super().__call__(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_block.py", line 406, in forward
[rank15]: checkpointed_result = self._checkpointed_forward(
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_block.py", line 206, in _checkpointed_forward
[rank15]: hidden_states, context = checkpoint_handler(custom(layer_idx, chunk_end))
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_block.py", line 189, in checkpoint_handler
[rank15]: return tensor_parallel.checkpoint(
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/tensor_parallel/random.py", line 576, in checkpoint
[rank15]: return CheckpointFunction.apply(function, distribute_saved_activations, *args)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/autograd/function.py", line 581, in apply
[rank15]: return super().apply(*args, **kwargs) # type: ignore[misc]
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/tensor_parallel/random.py", line 517, in forward
[rank15]: outputs = run_function(*args)
[rank15]: ^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_block.py", line 165, in wrapped_forward
[rank15]: return forward_func(
[rank15]: ^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_block.py", line 135, in custom_forward
[rank15]: hidden_states, context = self._layer_forward(
[rank15]: ^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_block.py", line 251, in _layer_forward
[rank15]: return layer(hidden_states=hidden_states, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/transformer_layer.py", line 1217, in __call__
[rank15]: return super().__call__(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/module.py", line 352, in __call__
[rank15]: return super().__call__(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_layer.py", line 364, in forward
[rank15]: output = self._forward_mlp(hidden_states, kwargs.get('inference_context', None), **mlp_kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/transformer_layer.py", line 756, in _forward_mlp
[rank15]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output, padding_mask=padding_mask)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/mlp.py", line 159, in forward
[rank15]: intermediate_parallel, bias_parallel = self.linear_fc1(hidden_states)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/tuners/lora.py", line 405, in forward
[rank15]: lora_B, TEGroupedLinear) else lora_B(lora_result)
[rank15]: ^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/tiger/ms-swift/swift/tuners/peft.py", line 268, in keep_device_forward
[rank15]: return self.forward_origin(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/tensor_parallel/layers.py", line 1022, in forward
[rank15]: output_parallel = self._forward_impl(
[rank15]: ^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/tensor_parallel/layers.py", line 952, in _forward_impl
[rank15]: return linear_with_grad_accumulation_and_async_allreduce(input, weight, *args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/tensor_parallel/layers.py", line 745, in linear_with_grad_accumulation_and_async_allreduce
[rank15]: return LinearWithGradAccumulationAndAsyncCommunication.apply(*args)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/autograd/function.py", line 581, in apply
[rank15]: return super().apply(*args, **kwargs) # type: ignore[misc]
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/amp/autocast_mode.py", line 527, in decorate_fwd
[rank15]: return fwd(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/tensor_parallel/layers.py", line 488, in forward
[rank15]: output = torch.matmul(total_input, weight.t())
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: RuntimeError: The Inner error is reported as above. The process exits for this inner error, and the current working operator name is aclnnFlashAttentionScore.
[rank15]: Since the operator is called asynchronously, the stacktrace may be inaccurate. If you want to get the accurate stacktrace, please set the environment variable ASCEND_LAUNCH_BLOCKING=1.
[rank15]: Note: ASCEND_LAUNCH_BLOCKING=1 will force ops to run in synchronous mode, resulting in performance degradation. Please unset ASCEND_LAUNCH_BLOCKING in time after debugging.
[rank15]: [ERROR] 2026-06-02-10:19:58 (PID:734915, Device:15, RankID:15) ERR00100 PTA call acl api failed.
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
Train: 0%| | 0/900 [00:45<?, ?it/s]
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
W0602 10:20:20.661000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734900 closing signal SIGTERM
W0602 10:20:20.663000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734901 closing signal SIGTERM
W0602 10:20:20.663000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734902 closing signal SIGTERM
W0602 10:20:20.664000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734903 closing signal SIGTERM
W0602 10:20:20.664000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734904 closing signal SIGTERM
W0602 10:20:20.664000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734905 closing signal SIGTERM
W0602 10:20:20.664000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734906 closing signal SIGTERM
W0602 10:20:20.664000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734907 closing signal SIGTERM
W0602 10:20:20.665000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734908 closing signal SIGTERM
W0602 10:20:20.665000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734910 closing signal SIGTERM
W0602 10:20:20.665000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734911 closing signal SIGTERM
W0602 10:20:20.665000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734912 closing signal SIGTERM
W0602 10:20:20.665000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734913 closing signal SIGTERM
W0602 10:20:20.666000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734914 closing signal SIGTERM
W0602 10:20:20.666000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734915 closing signal SIGTERM
E0602 10:20:21.515000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 9 (pid: 734909) of binary: /usr/local/bin/python
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/usr/local/lib/python3.11/site-packages/torch/distributed/run.py", line 940, in <module>
main()
File "/usr/local/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
return f(*args, **kwargs)
^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/torch/distributed/run.py", line 936, in main
run(args)
File "/usr/local/lib/python3.11/site-packages/torch/distributed/run.py", line 927, in run
elastic_launch(
File "/usr/local/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 156, in __call__
return launch_agent(self._config, self._entrypoint, list(args))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
============================================================
/opt/tiger/ms-swift/swift/cli/_megatron/sft.py FAILED
------------------------------------------------------------
Failures:
<NO_OTHER_FAILURES>
------------------------------------------------------------
Root Cause (first observed failure):
[0]:
time : 2026-06-02_10:20:20
host : g340-cd51-4900-6b09-b7c-e4cc-7104.byted.org
rank : 9 (local_rank: 9)
exitcode : 1 (pid: 734909)
error_file: <N/A>
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
============================================================
[ERROR] 2026-06-02-10:20:21 (PID:734873, Device:-1, RankID:-1) ERR99999 UNKNOWN applicaiton exception
点赞
收藏
回复
分享
举报
已解决
[rank15]: Traceback (most recent call last):
[rank15]: File "/opt/tiger/ms-swift/swift/cli/_megatron/sft.py", line 7, in <module>
[rank15]: megatron_sft_main()
[rank15]: File "/opt/tiger/ms-swift/swift/megatron/pipelines/train/sft.py", line 97, in megatron_sft_main
[rank15]: return MegatronSft(args).main()
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/tiger/ms-swift/swift/pipelines/base.py", line 52, in main
[rank15]: result = self.run()
[rank15]: ^^^^^^^^^^
[rank15]: File "/opt/tiger/ms-swift/swift/megatron/pipelines/train/sft.py", line 72, in run
[rank15]: trainer.train(train_dataset, val_dataset)
[rank15]: File "/opt/tiger/ms-swift/swift/megatron/trainers/base.py", line 636, in train
[rank15]: metrics, grad_norm, update_successful = self.train_step(train_data_iterator)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/tiger/ms-swift/swift/megatron/trainers/base.py", line 857, in train_step
[rank15]: metrics = forward_backward_func(
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/pipeline_parallel/schedules.py", line 656, in forward_backward_no_pipelining
[rank15]: output_tensor, num_tokens = forward_step(
[rank15]: ^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/pipeline_parallel/schedules.py", line 423, in forward_step
[rank15]: output_tensor, loss_func = forward_step_func(data_iterator, model)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/tiger/ms-swift/swift/megatron/trainers/trainer.py", line 124, in forward_step
[rank15]: output_tensor = model(**data)
[rank15]: ^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/distributed/data_parallel_base.py", line 22, in forward
[rank15]: return self.module(*inputs, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/module.py", line 489, in forward
[rank15]: outputs = self.module(*inputs, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/mm_gpt_model.py", line 95, in forward
[rank15]: return self.language_model(
[rank15]: ^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/gpt_model.py", line 310, in forward
[rank15]: decoder_output = self.decoder(
[rank15]: ^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/transformer_block.py", line 619, in __call__
[rank15]: return super().__call__(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/module.py", line 352, in __call__
[rank15]: return super().__call__(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_block.py", line 406, in forward
[rank15]: checkpointed_result = self._checkpointed_forward(
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_block.py", line 206, in _checkpointed_forward
[rank15]: hidden_states, context = checkpoint_handler(custom(layer_idx, chunk_end))
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_block.py", line 189, in checkpoint_handler
[rank15]: return tensor_parallel.checkpoint(
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/tensor_parallel/random.py", line 576, in checkpoint
[rank15]: return CheckpointFunction.apply(function, distribute_saved_activations, *args)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/autograd/function.py", line 581, in apply
[rank15]: return super().apply(*args, **kwargs) # type: ignore[misc]
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/tensor_parallel/random.py", line 517, in forward
[rank15]: outputs = run_function(*args)
[rank15]: ^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_block.py", line 165, in wrapped_forward
[rank15]: return forward_func(
[rank15]: ^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_block.py", line 135, in custom_forward
[rank15]: hidden_states, context = self._layer_forward(
[rank15]: ^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_block.py", line 251, in _layer_forward
[rank15]: return layer(hidden_states=hidden_states, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/transformer_layer.py", line 1217, in __call__
[rank15]: return super().__call__(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/module.py", line 352, in __call__
[rank15]: return super().__call__(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/model/modules/transformer_layer.py", line 364, in forward
[rank15]: output = self._forward_mlp(hidden_states, kwargs.get('inference_context', None), **mlp_kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/transformer_layer.py", line 756, in _forward_mlp
[rank15]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output, padding_mask=padding_mask)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/transformer/mlp.py", line 159, in forward
[rank15]: intermediate_parallel, bias_parallel = self.linear_fc1(hidden_states)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/mcore-bridge/src/mcore_bridge/tuners/lora.py", line 405, in forward
[rank15]: lora_B, TEGroupedLinear) else lora_B(lora_result)
[rank15]: ^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank15]: return self._call_impl(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank15]: return forward_call(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/tiger/ms-swift/swift/tuners/peft.py", line 268, in keep_device_forward
[rank15]: return self.forward_origin(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/tensor_parallel/layers.py", line 1022, in forward
[rank15]: output_parallel = self._forward_impl(
[rank15]: ^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/tensor_parallel/layers.py", line 952, in _forward_impl
[rank15]: return linear_with_grad_accumulation_and_async_allreduce(input, weight, *args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/tensor_parallel/layers.py", line 745, in linear_with_grad_accumulation_and_async_allreduce
[rank15]: return LinearWithGradAccumulationAndAsyncCommunication.apply(*args)
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/autograd/function.py", line 581, in apply
[rank15]: return super().apply(*args, **kwargs) # type: ignore[misc]
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/usr/local/lib/python3.11/site-packages/torch/amp/autocast_mode.py", line 527, in decorate_fwd
[rank15]: return fwd(*args, **kwargs)
[rank15]: ^^^^^^^^^^^^^^^^^^^^
[rank15]: File "/opt/Megatron-LM/megatron/core/tensor_parallel/layers.py", line 488, in forward
[rank15]: output = torch.matmul(total_input, weight.t())
[rank15]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank15]: RuntimeError: The Inner error is reported as above. The process exits for this inner error, and the current working operator name is aclnnFlashAttentionScore.
[rank15]: Since the operator is called asynchronously, the stacktrace may be inaccurate. If you want to get the accurate stacktrace, please set the environment variable ASCEND_LAUNCH_BLOCKING=1.
[rank15]: Note: ASCEND_LAUNCH_BLOCKING=1 will force ops to run in synchronous mode, resulting in performance degradation. Please unset ASCEND_LAUNCH_BLOCKING in time after debugging.
[rank15]: [ERROR] 2026-06-02-10:19:58 (PID:734915, Device:15, RankID:15) ERR00100 PTA call acl api failed.
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
Train: 0%| | 0/900 [00:45<?, ?it/s]
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
sys:1: ResourceWarning: unclosed <socket.socket fd=7, family=1, type=2, proto=0>
W0602 10:20:20.661000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734900 closing signal SIGTERM
W0602 10:20:20.663000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734901 closing signal SIGTERM
W0602 10:20:20.663000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734902 closing signal SIGTERM
W0602 10:20:20.664000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734903 closing signal SIGTERM
W0602 10:20:20.664000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734904 closing signal SIGTERM
W0602 10:20:20.664000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734905 closing signal SIGTERM
W0602 10:20:20.664000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734906 closing signal SIGTERM
W0602 10:20:20.664000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734907 closing signal SIGTERM
W0602 10:20:20.665000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734908 closing signal SIGTERM
W0602 10:20:20.665000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734910 closing signal SIGTERM
W0602 10:20:20.665000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734911 closing signal SIGTERM
W0602 10:20:20.665000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734912 closing signal SIGTERM
W0602 10:20:20.665000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734913 closing signal SIGTERM
W0602 10:20:20.666000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734914 closing signal SIGTERM
W0602 10:20:20.666000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 734915 closing signal SIGTERM
E0602 10:20:21.515000 734873 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 9 (pid: 734909) of binary: /usr/local/bin/python
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/usr/local/lib/python3.11/site-packages/torch/distributed/run.py", line 940, in <module>
main()
File "/usr/local/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
return f(*args, **kwargs)
^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/torch/distributed/run.py", line 936, in main
run(args)
File "/usr/local/lib/python3.11/site-packages/torch/distributed/run.py", line 927, in run
elastic_launch(
File "/usr/local/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 156, in __call__
return launch_agent(self._config, self._entrypoint, list(args))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
============================================================
/opt/tiger/ms-swift/swift/cli/_megatron/sft.py FAILED
------------------------------------------------------------
Failures:
<NO_OTHER_FAILURES>
------------------------------------------------------------
Root Cause (first observed failure):
[0]:
time : 2026-06-02_10:20:20
host : g340-cd51-4900-6b09-b7c-e4cc-7104.byted.org
rank : 9 (local_rank: 9)
exitcode : 1 (pid: 734909)
error_file: <N/A>
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
============================================================
[ERROR] 2026-06-02-10:20:21 (PID:734873, Device:-1, RankID:-1) ERR99999 UNKNOWN applicaiton exception