NPROC_PER_NODE=2 \
ASCEND_RT_VISIBLE_DEVICES=0,1 \
swift sft \
--model /home/shengsiyuan/OpenGVLab/InternVL3-2B \
--model_type internvl2_5 \
--train_type lora \
--dataset /home/shengsiyuan/dataset/mllm_data/mllm_data.json \
--torch_dtype bfloat16 \
--num_train_epochs 2 \
--per_device_train_batch_size 1 \
--per_device_eval_batch_size 1 \
--learning_rate 1e-4 \
--lora_rank 8 \
--lora_alpha 32 \
--target_modules all-linear \
--gradient_accumulation_steps 1 \
--eval_steps 50 \
--save_steps 50 \
--save_total_limit 5 \
--logging_steps 5 \
--max_length 1024 \
--output_dir internvlsoutput \
--system 'You are a medical domain expert assistant.' \
--warmup_ratio 0.05 \
--dataloader_num_workers 4 \
--model_author swift \
--model_name swift-robot
将上述指令删除NPROC_PER_NODE，修改ASCEND_RT_VISIBLE_DEVICES=0，可成功启动VL模型的微调，但是多卡微调时一直报错，使用官方gitee提供的torch2.5.1，torch_npu 2.5.1，CANN版本
package_name=Ascend-cann-toolkit
version=8.1.RC1
innerversion=V100R001C21SPC001B238
compatible_version=[V100R001C15],[V100R001C18],[V100R001C19],[V100R001C20],[V100R001C21]
arch=aarch64
os=linux
path=/usr/local/Cann/ascend-toolkit/8.1.RC1/aarch64-linux
具体的报错信息如下：
[INFO:swift] Loading the model using model_dir: /home/shengsiyuan/OpenGVLab/InternVL3-2B
[INFO:swift] model_kwargs: {'device_map': 'npu:0'}
FlashAttention2 is not installed.
[rank0]: Traceback (most recent call last):
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/cli/sft.py", line 10, in <module>
[rank0]: sft_main()
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/train/sft.py", line 265, in sft_main
[rank0]: return SwiftSft(args).main()
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/train/sft.py", line 30, in __init__
[rank0]: self._prepare_model_tokenizer()
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/train/sft.py", line 61, in _prepare_model_tokenizer
[rank0]: self.model, self.processor = args.get_model_processor()
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/argument/base_args/base_args.py", line 275, in get_model_processor
[rank0]: return get_model_tokenizer(**kwargs)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/register.py", line 532, in get_model_tokenizer
[rank0]: model, processor = get_function(model_dir, model_info, model_kwargs, load_model, **kwargs)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/model/internlm.py", line 129, in get_model_tokenizer_internvl
[rank0]: model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/register.py", line 236, in get_model_tokenizer_with_flash_attn
[rank0]: return get_model_tokenizer_from_local(model_dir, model_info, model_kwargs, load_model, **kwargs)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/register.py", line 209, in get_model_tokenizer_from_local
[rank0]: model = automodel_class.from_pretrained(
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 568, in from_pretrained
[rank0]: return model_class.from_pretrained(
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/patcher.py", line 285, in _new_from_pretrained
[rank0]: return from_pretrained(cls, *args, **kwargs)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 272, in _wrapper
[rank0]: return func(*args, **kwargs)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 4455, in from_pretrained
[rank0]: ) = cls._load_pretrained_model(
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 4841, in _load_pretrained_model
[rank0]: caching_allocator_warmup(model_to_load, expanded_device_map)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 5829, in caching_allocator_warmup
[rank0]: re.compile("|".join([re.escape(plan) for plan in model._tp_plan]))
[rank0]: TypeError: 'NoneType' object is not iterable
FlashAttention2 is not installed.
[rank1]: Traceback (most recent call last):
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/cli/sft.py", line 10, in <module>
[rank1]: sft_main()
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/train/sft.py", line 265, in sft_main
[rank1]: return SwiftSft(args).main()
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/train/sft.py", line 30, in __init__
[rank1]: self._prepare_model_tokenizer()
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/train/sft.py", line 61, in _prepare_model_tokenizer
[rank1]: self.model, self.processor = args.get_model_processor()
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/argument/base_args/base_args.py", line 275, in get_model_processor
[rank1]: return get_model_tokenizer(**kwargs)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/register.py", line 532, in get_model_tokenizer
[rank1]: model, processor = get_function(model_dir, model_info, model_kwargs, load_model, **kwargs)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/model/internlm.py", line 129, in get_model_tokenizer_internvl
[rank1]: model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/register.py", line 236, in get_model_tokenizer_with_flash_attn
[rank1]: return get_model_tokenizer_from_local(model_dir, model_info, model_kwargs, load_model, **kwargs)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/register.py", line 209, in get_model_tokenizer_from_local
[rank1]: model = automodel_class.from_pretrained(
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 568, in from_pretrained
[rank1]: return model_class.from_pretrained(
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/patcher.py", line 285, in _new_from_pretrained
[rank1]: return from_pretrained(cls, *args, **kwargs)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 272, in _wrapper
[rank1]: return func(*args, **kwargs)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 4455, in from_pretrained
[rank1]: ) = cls._load_pretrained_model(
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 4841, in _load_pretrained_model
[rank1]: caching_allocator_warmup(model_to_load, expanded_device_map)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 5829, in caching_allocator_warmup
[rank1]: re.compile("|".join([re.escape(plan) for plan in model._tp_plan]))
[rank1]: TypeError: 'NoneType' object is not iterable
[ERROR] 2025-06-11-17:00:01 (PID:8716, Device:1, RankID:-1) ERR99999 UNKNOWN applicaiton exception
[ERROR] 2025-06-11-17:00:01 (PID:8715, Device:0, RankID:-1) ERR99999 UNKNOWN applicaiton exception
E0611 17:00:06.395000 8647 site-packages/torch/distributed/elastic/multiprocessing/api.py:869] failed (exitcode: 1) local_rank: 0 (pid: 8715) of binary: /root/miniconda3/envs/cmri/bin/python3.10
Traceback (most recent call last):
File "/root/miniconda3/envs/cmri/lib/python3.10/runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/root/miniconda3/envs/cmri/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/torch/distributed/run.py", line 923, in <module>
main()
File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
return f(*args, **kwargs)
File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/torch/distributed/run.py", line 919, in main
run(args)
File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/torch/distributed/run.py", line 910, in run
elastic_launch(
File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 138, in __call__
return launch_agent(self._config, self._entrypoint, list(args))
File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 269, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
============================================================
/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/cli/sft.py FAILED
------------------------------------------------------------
Failures:
[1]:
time : 2025-06-11_17:00:06
host : localhost.localdomain
rank : 1 (local_rank: 1)
exitcode : 1 (pid: 8716)
error_file: <N/A>
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
------------------------------------------------------------
Root Cause (first observed failure):
[0]:
time : 2025-06-11_17:00:06
host : localhost.localdomain
rank : 0 (local_rank: 0)
exitcode : 1 (pid: 8715)
error_file: <N/A>
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
============================================================
[ERROR] 2025-06-11-17:00:06 (PID:8647, Device:-1, RankID:-1) ERR99999 UNKNOWN applicaiton exception
点赞
收藏
回复
分享
举报
已解决
ASCEND_RT_VISIBLE_DEVICES=0,1 \
swift sft \
--model /home/shengsiyuan/OpenGVLab/InternVL3-2B \
--model_type internvl2_5 \
--train_type lora \
--dataset /home/shengsiyuan/dataset/mllm_data/mllm_data.json \
--torch_dtype bfloat16 \
--num_train_epochs 2 \
--per_device_train_batch_size 1 \
--per_device_eval_batch_size 1 \
--learning_rate 1e-4 \
--lora_rank 8 \
--lora_alpha 32 \
--target_modules all-linear \
--gradient_accumulation_steps 1 \
--eval_steps 50 \
--save_steps 50 \
--save_total_limit 5 \
--logging_steps 5 \
--max_length 1024 \
--output_dir internvlsoutput \
--system 'You are a medical domain expert assistant.' \
--warmup_ratio 0.05 \
--dataloader_num_workers 4 \
--model_author swift \
--model_name swift-robot
将上述指令删除NPROC_PER_NODE，修改ASCEND_RT_VISIBLE_DEVICES=0，可成功启动VL模型的微调，但是多卡微调时一直报错，使用官方gitee提供的torch2.5.1，torch_npu 2.5.1，CANN版本
package_name=Ascend-cann-toolkit
version=8.1.RC1
innerversion=V100R001C21SPC001B238
compatible_version=[V100R001C15],[V100R001C18],[V100R001C19],[V100R001C20],[V100R001C21]
arch=aarch64
os=linux
path=/usr/local/Cann/ascend-toolkit/8.1.RC1/aarch64-linux
具体的报错信息如下：
[INFO:swift] Loading the model using model_dir: /home/shengsiyuan/OpenGVLab/InternVL3-2B
[INFO:swift] model_kwargs: {'device_map': 'npu:0'}
FlashAttention2 is not installed.
[rank0]: Traceback (most recent call last):
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/cli/sft.py", line 10, in <module>
[rank0]: sft_main()
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/train/sft.py", line 265, in sft_main
[rank0]: return SwiftSft(args).main()
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/train/sft.py", line 30, in __init__
[rank0]: self._prepare_model_tokenizer()
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/train/sft.py", line 61, in _prepare_model_tokenizer
[rank0]: self.model, self.processor = args.get_model_processor()
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/argument/base_args/base_args.py", line 275, in get_model_processor
[rank0]: return get_model_tokenizer(**kwargs)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/register.py", line 532, in get_model_tokenizer
[rank0]: model, processor = get_function(model_dir, model_info, model_kwargs, load_model, **kwargs)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/model/internlm.py", line 129, in get_model_tokenizer_internvl
[rank0]: model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/register.py", line 236, in get_model_tokenizer_with_flash_attn
[rank0]: return get_model_tokenizer_from_local(model_dir, model_info, model_kwargs, load_model, **kwargs)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/register.py", line 209, in get_model_tokenizer_from_local
[rank0]: model = automodel_class.from_pretrained(
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 568, in from_pretrained
[rank0]: return model_class.from_pretrained(
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/patcher.py", line 285, in _new_from_pretrained
[rank0]: return from_pretrained(cls, *args, **kwargs)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 272, in _wrapper
[rank0]: return func(*args, **kwargs)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 4455, in from_pretrained
[rank0]: ) = cls._load_pretrained_model(
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 4841, in _load_pretrained_model
[rank0]: caching_allocator_warmup(model_to_load, expanded_device_map)
[rank0]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 5829, in caching_allocator_warmup
[rank0]: re.compile("|".join([re.escape(plan) for plan in model._tp_plan]))
[rank0]: TypeError: 'NoneType' object is not iterable
FlashAttention2 is not installed.
[rank1]: Traceback (most recent call last):
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/cli/sft.py", line 10, in <module>
[rank1]: sft_main()
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/train/sft.py", line 265, in sft_main
[rank1]: return SwiftSft(args).main()
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/train/sft.py", line 30, in __init__
[rank1]: self._prepare_model_tokenizer()
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/train/sft.py", line 61, in _prepare_model_tokenizer
[rank1]: self.model, self.processor = args.get_model_processor()
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/argument/base_args/base_args.py", line 275, in get_model_processor
[rank1]: return get_model_tokenizer(**kwargs)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/register.py", line 532, in get_model_tokenizer
[rank1]: model, processor = get_function(model_dir, model_info, model_kwargs, load_model, **kwargs)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/model/internlm.py", line 129, in get_model_tokenizer_internvl
[rank1]: model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/register.py", line 236, in get_model_tokenizer_with_flash_attn
[rank1]: return get_model_tokenizer_from_local(model_dir, model_info, model_kwargs, load_model, **kwargs)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/register.py", line 209, in get_model_tokenizer_from_local
[rank1]: model = automodel_class.from_pretrained(
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 568, in from_pretrained
[rank1]: return model_class.from_pretrained(
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/llm/model/patcher.py", line 285, in _new_from_pretrained
[rank1]: return from_pretrained(cls, *args, **kwargs)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 272, in _wrapper
[rank1]: return func(*args, **kwargs)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 4455, in from_pretrained
[rank1]: ) = cls._load_pretrained_model(
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 4841, in _load_pretrained_model
[rank1]: caching_allocator_warmup(model_to_load, expanded_device_map)
[rank1]: File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/transformers/modeling_utils.py", line 5829, in caching_allocator_warmup
[rank1]: re.compile("|".join([re.escape(plan) for plan in model._tp_plan]))
[rank1]: TypeError: 'NoneType' object is not iterable
[ERROR] 2025-06-11-17:00:01 (PID:8716, Device:1, RankID:-1) ERR99999 UNKNOWN applicaiton exception
[ERROR] 2025-06-11-17:00:01 (PID:8715, Device:0, RankID:-1) ERR99999 UNKNOWN applicaiton exception
E0611 17:00:06.395000 8647 site-packages/torch/distributed/elastic/multiprocessing/api.py:869] failed (exitcode: 1) local_rank: 0 (pid: 8715) of binary: /root/miniconda3/envs/cmri/bin/python3.10
Traceback (most recent call last):
File "/root/miniconda3/envs/cmri/lib/python3.10/runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/root/miniconda3/envs/cmri/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/torch/distributed/run.py", line 923, in <module>
main()
File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
return f(*args, **kwargs)
File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/torch/distributed/run.py", line 919, in main
run(args)
File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/torch/distributed/run.py", line 910, in run
elastic_launch(
File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 138, in __call__
return launch_agent(self._config, self._entrypoint, list(args))
File "/root/miniconda3/envs/cmri/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 269, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
============================================================
/root/miniconda3/envs/cmri/lib/python3.10/site-packages/swift/cli/sft.py FAILED
------------------------------------------------------------
Failures:
[1]:
time : 2025-06-11_17:00:06
host : localhost.localdomain
rank : 1 (local_rank: 1)
exitcode : 1 (pid: 8716)
error_file: <N/A>
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
------------------------------------------------------------
Root Cause (first observed failure):
[0]:
time : 2025-06-11_17:00:06
host : localhost.localdomain
rank : 0 (local_rank: 0)
exitcode : 1 (pid: 8715)
error_file: <N/A>
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
============================================================
[ERROR] 2025-06-11-17:00:06 (PID:8647, Device:-1, RankID:-1) ERR99999 UNKNOWN applicaiton exception