用910B部署Qwen2.5VL-32B宕机，从MindIE日志上看是npu内存爆了。怀疑是其中前处理的vit模块没有队列机制导致的，正常跑其他NLP模型请求多了只会等待。经多次测试，发现一旦并发多了直接就OOM。

具体报错信息：

[ERROR] [standard_model.py:197] : [Model]    >>> global rank-0 Execute type:1, Exception:NPU out of memory. Tried to allocate 176.00 MiB (NPU 4; 29.50 GiB total capacity; 27.94 GiB already allocated; 27.94 GiB current active; 27.22 MiB free; 28.41 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.

Traceback (most recent call last):

  File "/usr/local/lib/python3.11/site-packages/model_wrapper/standard_model.py", line 185, in execute

    return self._prefill(requests)

           ^^^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/lib/python3.11/site-packages/model_wrapper/standard_model.py", line 430, in _prefill

    return self.__generate(requests, is_prefill=True, is_mix=False)

           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/lib/python3.11/site-packages/model_wrapper/standard_model.py", line 578, in __generate

    generate_output = self.__handle_requests(requests, is_prefill, is_mix)

                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/lib/python3.11/site-packages/model_wrapper/standard_model.py", line 939, in __handle_requests

    generate_output = self.generator.generate_token(metadata)

                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 386, in generate_token

    raise e

  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 367, in generate_token

    generation_output = self.plugin.generate_token(input_metadata)

                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/lib/python3.11/site-packages/mindie_llm/utils/decorators/time_decorator.py", line 91, in wrapper

    return func(*args, **kwargs)

           ^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/plugins/plugin_manager.py", line 115, in generate_token

    result = self.generator_backend.forward(model_inputs, q_lens=self.plugin_data_param.q_len,

             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/lib/python3.11/site-packages/mindie_llm/utils/decorators/time_decorator.py", line 69, in wrapper

    return func(*args, **kwargs)

           ^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 230, in forward

    logits = self._forward(model_inputs, **kwargs)

             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 621, in _forward

    logits = self.model_wrapper.forward(model_inputs, self.cache_pool.npu_cache, **kwargs)

             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/lib/python3.11/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 125, in forward

    result = self.forward_from_model_inputs(model_inputs, npu_cache, **kwargs)

             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/lib/python3.11/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 197, in forward_from_model_inputs

    result = self.forward_tensor(

             ^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/lib/python3.11/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 237, in forward_tensor

    result = self.model_runner.forward(

             ^^^^^^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/Ascend/atb-models/atb_llm/runner/model_runner.py", line 310, in forward

    res = self.model.forward(**kwargs)

          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2_vl/flash_causal_qwen2_vl.py", line 118, in forward

    inputs_embeds, image_grid_thw, video_grid_thw, second_per_grid_ts = self.prepare_prefill_token_service(

                                                                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

  File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2_vl/flash_causal_qwen2_vl.py", line 195, in prepare_prefill_token_service

    image_pixel = image_pixel * self.norm_rescale_w + self.norm_rescale_b