1、mindie容器镜像信息：
2、生成容器命令
3、容器中npu-smi info情况
4、宿主服务器配置：
5、容器中mindie运行信息
[root@32e5aaa8f3ae bin]# ./mindieservice_daemon
[2025-03-19 18:57:55,193] [8597] [140397127415488] [llm] [INFO] [cpu_binding.py-212] : rank_id: 1, device_id: 1, numa_id: 1, shard_devices: [1], cpus: [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95]
[2025-03-19 18:57:55,195] [8597] [140397127415488] [llm] [INFO] [cpu_binding.py-238] : process 8597, new_affinity is [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95], cpu count 48
[2025-03-19 18:57:55,708] [8597] [140397127415488] [llm] [INFO] [logging.py-331] : model_runner.quantize: None, model_runner.kv_quant_type: None, model_runner.fa_quant_type: None, model_runner.dtype: torch.float16
[2025-03-19 18:57:59,446] [8597] [140397127415488] [llm] [INFO] [dist.py-81] : initialize_distributed has been Set
[2025-03-19 18:57:59,447] [8597] [140397127415488] [llm] [INFO] [logging.py-331] : init tokenizer done: LlamaTokenizerFast(name_or_path='/models/DeepSeek-R1-Distill-Qwen-1.5B', vocab_size=151643, model_max_length=16384, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'bos_token': '<\uff5cbegin\u2581of\u2581sentence\uff5c>', 'eos_token': '<\uff5cend\u2581of\u2581sentence\uff5c>', 'pad_token': '<\uff5cend\u2581of\u2581sentence\uff5c>'}, clean_up_tokenization_spaces=False), added_tokens_decoder={
151643: AddedToken("<\uff5cend\u2581of\u2581sentence\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151644: AddedToken("<\uff5cUser\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151645: AddedToken("<\uff5cAssistant\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151646: AddedToken("<\uff5cbegin\u2581of\u2581sentence\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151647: AddedToken("<|EOT|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151648: AddedToken("<think>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151649: AddedToken("</think>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151650: AddedToken("<|quad_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151651: AddedToken("<|quad_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151652: AddedToken("<|vision_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151653: AddedToken("<|vision_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151654: AddedToken("<|vision_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151655: AddedToken("<|image_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151656: AddedToken("<|video_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151657: AddedToken("<tool_call>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151658: AddedToken("</tool_call>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151659: AddedToken("<|fim_prefix|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151660: AddedToken("<|fim_middle|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151661: AddedToken("<|fim_suffix|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151662: AddedToken("<|fim_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151663: AddedToken("<|repo_name|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151664: AddedToken("<|file_sep|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
}
--- Logging error ---
Traceback (most recent call last):
File "/usr/lib64/python3.11/logging/__init__.py", line 1112, in emit
stream.write(msg + self.terminator)
UnicodeEncodeError: 'ascii' codec can't encode character '\uff5c' in position 311: ordinal not in range(128)
Call stack:
[2025-03-19 18:57:59,451] [8597] [140397127415488] [llm] [INFO] [logging.py-331] : NPUSocInfo(soc_name='', soc_version=103, need_nz=True, matmul_nd_nz=False)
File "/usr/lib64/python3.11/threading.py", line 1002, in _bootstrap
self._bootstrap_inner()
File "/usr/lib64/python3.11/threading.py", line 1045, in _bootstrap_inner
self.run()
File "/usr/lib64/python3.11/threading.py", line 982, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/Ascend/atb-models/atb_llm/utils/log/multiprocess_logging_handler.py", line 80, in receive
self.log_handler.emit(record)
Message: 'init tokenizer done: LlamaTokenizerFast(name_or_path=\'/models/DeepSeek-R1-Distill-Qwen-1.5B\', vocab_size=151643, model_max_length=16384, is_fast=True, padding_side=\'left\', truncation_side=\'right\', special_tokens={\'bos_token\': \'<\uff5cbegin\u2581of\u2581sentence\uff5c>\', \'eos_token\': \'<\uff5cend\u2581of\u2581sentence\uff5c>\', \'pad_token\': \'<\uff5cend\u2581of\u2581sentence\uff5c>\'}, clean_up_tokenization_spaces=False), added_tokens_decoder={\n\t151643: AddedToken("<\uff5cend\u2581of\u2581sentence\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151644: AddedToken("<\uff5cUser\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151645: AddedToken("<\uff5cAssistant\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151646: AddedToken("<\uff5cbegin\u2581of\u2581sentence\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151647: AddedToken("<|EOT|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151648: AddedToken("<think>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151649: AddedToken("</think>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151650: AddedToken("<|quad_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151651: AddedToken("<|quad_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151652: AddedToken("<|vision_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151653: AddedToken("<|vision_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151654: AddedToken("<|vision_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151655: AddedToken("<|image_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151656: AddedToken("<|video_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151657: AddedToken("<tool_call>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151658: AddedToken("</tool_call>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151659: AddedToken("<|fim_prefix|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151660: AddedToken("<|fim_middle|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151661: AddedToken("<|fim_suffix|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151662: AddedToken("<|fim_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151663: AddedToken("<|repo_name|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151664: AddedToken("<|file_sep|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n}'
Arguments: ()
Process ForkServerProcess-9:
Process ForkServerProcess-5:
Process ForkServerProcess-3:
Process ForkServerProcess-2:
Process ForkServerProcess-7:
Process ForkServerProcess-8:
Process ForkServerProcess-4:
Process ForkServerProcess-6:
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
During handling of the above exception, another exception occurred:
During handling of the above exception, another exception occurred:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
Traceback (most recent call last):
During handling of the above exception, another exception occurred:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
During handling of the above exception, another exception occurred:
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "<string>", line 2, in append
File "<string>", line 2, in append
File "<string>", line 2, in append
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "<string>", line 2, in append
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<string>", line 2, in append
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
ConnectionRefusedError: [Errno 111] Connection refused
ConnectionRefusedError: [Errno 111] Connection refused
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "<string>", line 2, in append
ConnectionRefusedError: [Errno 111] Connection refused
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ConnectionRefusedError: [Errno 111] Connection refused
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "<string>", line 2, in append
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
ConnectionRefusedError: [Errno 111] Connection refused
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ConnectionRefusedError: [Errno 111] Connection refused
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
File "<string>", line 2, in append
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ConnectionRefusedError: [Errno 111] Connection refused
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
ConnectionRefusedError: [Errno 111] Connection refused
/usr/lib64/python3.11/multiprocessing/resource_tracker.py:254: UserWarning: resource_tracker: There appear to be 30 leaked semaphore objects to clean up at shutdown
warnings.warn('resource_tracker: There appear to be %d '
Daemon is killing...
Killed
6、容器中mindie config.json文件内容，黄色部分是修改的内容
点赞
收藏
回复
分享
举报
已解决
1、mindie容器镜像信息：
镜像版本：2.0.T3.1-800i-a2-py311-openeuler24.03-lts
# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
swr.cn-south-1.myhuaweicloud.com/ascendhub/mindie 2.0.T3.1-800I-A2-py311-openeuler24.03-lts afc648a904f9 5 weeks ago 17.6GB
2、生成容器命令
docker run -it -d -p 1025:1025 --shm-size=4g --name mindie --device=/dev/davinci_manager --device=/dev/hisi_hdc --device=/dev/devmm_svm --device=/dev/davinci1 --device=/dev/davinci4 -v /usr/local/Ascend/driver:/usr/local/Ascend/driver:ro -v /usr/local/sbin:/usr/local/sbin:ro -v /models:/models:ro afc648a904f9
3、容器中npu-smi info情况
4、宿主服务器配置：
型号：P_A800-3010
配置：intel xeon gold 5220r，内存 512G，2块ascend 910proB，硬盘：2块800G固态
操作系统：ubuntu 22.04.5 服务器版
python: 3.10.12
pytorch：2.1
torch-npu：2.1.0.post10
docker：28.0.1
docker compose：2.33.1
安装的软件包版本：
Ascend-hdk-910-npu-driver_24.1.rc2_linux-x86-64.run
Ascend-hdk-910-npu-firmware_7.3.0.1.231.run
Ascend-cann-nnal_8.0.0_linux-x86_64.run
Ascend-cann-kernels-910_8.0.0_linux-x86_64.run
Ascend-cann-toolkit_8.0.0_linux-x86_64.run
5、容器中mindie运行信息
[root@32e5aaa8f3ae bin]# ./mindieservice_daemon
[2025-03-19 18:57:55,193] [8597] [140397127415488] [llm] [INFO] [cpu_binding.py-212] : rank_id: 1, device_id: 1, numa_id: 1, shard_devices: [1], cpus: [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95]
[2025-03-19 18:57:55,195] [8597] [140397127415488] [llm] [INFO] [cpu_binding.py-238] : process 8597, new_affinity is [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95], cpu count 48
[2025-03-19 18:57:55,708] [8597] [140397127415488] [llm] [INFO] [logging.py-331] : model_runner.quantize: None, model_runner.kv_quant_type: None, model_runner.fa_quant_type: None, model_runner.dtype: torch.float16
[2025-03-19 18:57:59,446] [8597] [140397127415488] [llm] [INFO] [dist.py-81] : initialize_distributed has been Set
[2025-03-19 18:57:59,447] [8597] [140397127415488] [llm] [INFO] [logging.py-331] : init tokenizer done: LlamaTokenizerFast(name_or_path='/models/DeepSeek-R1-Distill-Qwen-1.5B', vocab_size=151643, model_max_length=16384, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'bos_token': '<\uff5cbegin\u2581of\u2581sentence\uff5c>', 'eos_token': '<\uff5cend\u2581of\u2581sentence\uff5c>', 'pad_token': '<\uff5cend\u2581of\u2581sentence\uff5c>'}, clean_up_tokenization_spaces=False), added_tokens_decoder={
151643: AddedToken("<\uff5cend\u2581of\u2581sentence\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151644: AddedToken("<\uff5cUser\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151645: AddedToken("<\uff5cAssistant\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151646: AddedToken("<\uff5cbegin\u2581of\u2581sentence\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151647: AddedToken("<|EOT|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151648: AddedToken("<think>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151649: AddedToken("</think>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151650: AddedToken("<|quad_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151651: AddedToken("<|quad_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151652: AddedToken("<|vision_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151653: AddedToken("<|vision_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151654: AddedToken("<|vision_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151655: AddedToken("<|image_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151656: AddedToken("<|video_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
151657: AddedToken("<tool_call>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151658: AddedToken("</tool_call>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151659: AddedToken("<|fim_prefix|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151660: AddedToken("<|fim_middle|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151661: AddedToken("<|fim_suffix|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151662: AddedToken("<|fim_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151663: AddedToken("<|repo_name|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
151664: AddedToken("<|file_sep|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),
}
--- Logging error ---
Traceback (most recent call last):
File "/usr/lib64/python3.11/logging/__init__.py", line 1112, in emit
stream.write(msg + self.terminator)
UnicodeEncodeError: 'ascii' codec can't encode character '\uff5c' in position 311: ordinal not in range(128)
Call stack:
[2025-03-19 18:57:59,451] [8597] [140397127415488] [llm] [INFO] [logging.py-331] : NPUSocInfo(soc_name='', soc_version=103, need_nz=True, matmul_nd_nz=False)
File "/usr/lib64/python3.11/threading.py", line 1002, in _bootstrap
self._bootstrap_inner()
File "/usr/lib64/python3.11/threading.py", line 1045, in _bootstrap_inner
self.run()
File "/usr/lib64/python3.11/threading.py", line 982, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/Ascend/atb-models/atb_llm/utils/log/multiprocess_logging_handler.py", line 80, in receive
self.log_handler.emit(record)
Message: 'init tokenizer done: LlamaTokenizerFast(name_or_path=\'/models/DeepSeek-R1-Distill-Qwen-1.5B\', vocab_size=151643, model_max_length=16384, is_fast=True, padding_side=\'left\', truncation_side=\'right\', special_tokens={\'bos_token\': \'<\uff5cbegin\u2581of\u2581sentence\uff5c>\', \'eos_token\': \'<\uff5cend\u2581of\u2581sentence\uff5c>\', \'pad_token\': \'<\uff5cend\u2581of\u2581sentence\uff5c>\'}, clean_up_tokenization_spaces=False), added_tokens_decoder={\n\t151643: AddedToken("<\uff5cend\u2581of\u2581sentence\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151644: AddedToken("<\uff5cUser\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151645: AddedToken("<\uff5cAssistant\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151646: AddedToken("<\uff5cbegin\u2581of\u2581sentence\uff5c>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151647: AddedToken("<|EOT|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151648: AddedToken("<think>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151649: AddedToken("</think>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151650: AddedToken("<|quad_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151651: AddedToken("<|quad_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151652: AddedToken("<|vision_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151653: AddedToken("<|vision_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151654: AddedToken("<|vision_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151655: AddedToken("<|image_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151656: AddedToken("<|video_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n\t151657: AddedToken("<tool_call>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151658: AddedToken("</tool_call>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151659: AddedToken("<|fim_prefix|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151660: AddedToken("<|fim_middle|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151661: AddedToken("<|fim_suffix|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151662: AddedToken("<|fim_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151663: AddedToken("<|repo_name|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n\t151664: AddedToken("<|file_sep|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n}'
Arguments: ()
Process ForkServerProcess-9:
Process ForkServerProcess-5:
Process ForkServerProcess-3:
Process ForkServerProcess-2:
Process ForkServerProcess-7:
Process ForkServerProcess-8:
Process ForkServerProcess-4:
Process ForkServerProcess-6:
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
During handling of the above exception, another exception occurred:
During handling of the above exception, another exception occurred:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
Traceback (most recent call last):
During handling of the above exception, another exception occurred:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
During handling of the above exception, another exception occurred:
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 814, in _callmethod
conn = self._tls.connection
^^^^^^^^^^^^^^^^^^^^
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "<string>", line 2, in append
File "<string>", line 2, in append
File "<string>", line 2, in append
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "<string>", line 2, in append
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<string>", line 2, in append
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
ConnectionRefusedError: [Errno 111] Connection refused
ConnectionRefusedError: [Errno 111] Connection refused
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "<string>", line 2, in append
ConnectionRefusedError: [Errno 111] Connection refused
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/lib64/python3.11/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ConnectionRefusedError: [Errno 111] Connection refused
File "/usr/lib64/python3.11/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "<string>", line 2, in append
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
ConnectionRefusedError: [Errno 111] Connection refused
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 71, in wrapper
raise exp
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 63, in wrapper
func(*args, **kwargs)
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ConnectionRefusedError: [Errno 111] Connection refused
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 264, in task_distribute
resource_proxy[SUB_PROCESS_STATE].append(True)
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
File "<string>", line 2, in append
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 818, in _callmethod
self._connect()
File "/usr/lib64/python3.11/multiprocessing/managers.py", line 805, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ConnectionRefusedError: [Errno 111] Connection refused
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 518, in Client
c = SocketClient(address)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/multiprocessing/connection.py", line 646, in SocketClient
s.connect(address)
ConnectionRefusedError: [Errno 111] Connection refused
/usr/lib64/python3.11/multiprocessing/resource_tracker.py:254: UserWarning: resource_tracker: There appear to be 30 leaked semaphore objects to clean up at shutdown
warnings.warn('resource_tracker: There appear to be %d '
Daemon is killing...
Killed
6、容器中mindie config.json文件内容，黄色部分是修改的内容
{
"Version" : "1.0.0",
"LogConfig" :
{
"logLevel" : "Info",
"logFileSize" : 20,
"logFileNum" : 20,
"logPath" : "logs/mindie-server.log"
},
"ServerConfig" :
{
"ipAddress" : "127.0.0.1",
"managementIpAddress" : "127.0.0.2",
"port" : 1025,
"managementPort" : 1026,
"metricsPort" : 1027,
"allowAllZeroIpListening" : false,
"maxLinkNum" : 1000,
"httpsEnabled" : false,
"fullTextEnabled" : false,
"tlsCaPath" : "security/ca/",
"tlsCaFile" : ["ca.pem"],
"tlsCert" : "security/certs/server.pem",
"tlsPk" : "security/keys/server.key.pem",
"tlsPkPwd" : "security/pass/key_pwd.txt",
"tlsCrlPath" : "security/certs/",
"tlsCrlFiles" : ["server_crl.pem"],
"managementTlsCaFile" : ["management_ca.pem"],
"managementTlsCert" : "security/certs/management/server.pem",
"managementTlsPk" : "security/keys/management/server.key.pem",
"managementTlsPkPwd" : "security/pass/management/key_pwd.txt",
"managementTlsCrlPath" : "security/management/certs/",
"managementTlsCrlFiles" : ["server_crl.pem"],
"kmcKsfMaster" : "tools/pmt/master/ksfa",
"kmcKsfStandby" : "tools/pmt/standby/ksfb",
"inferMode" : "standard",
"interCommTLSEnabled" : true,
"interCommPort" : 1121,
"interCommTlsCaPath" : "security/grpc/ca/",
"interCommTlsCaFiles" : ["ca.pem"],
"interCommTlsCert" : "security/grpc/certs/server.pem",
"interCommPk" : "security/grpc/keys/server.key.pem",
"interCommPkPwd" : "security/grpc/pass/key_pwd.txt",
"interCommTlsCrlPath" : "security/grpc/certs/",
"interCommTlsCrlFiles" : ["server_crl.pem"],
"openAiSupport" : "vllm"
},
"BackendConfig" : {
"backendName" : "mindieservice_llm_engine",
"modelInstanceNumber" : 1,
"npuDeviceIds" : [[1]],
"tokenizerProcessNumber" : 8,
"multiNodesInferEnabled" : false,
"multiNodesInferPort" : 1120,
"interNodeTLSEnabled" : true,
"interNodeTlsCaPath" : "security/grpc/ca/",
"interNodeTlsCaFiles" : ["ca.pem"],
"interNodeTlsCert" : "security/grpc/certs/server.pem",
"interNodeTlsPk" : "security/grpc/keys/server.key.pem",
"interNodeTlsPkPwd" : "security/grpc/pass/mindie_server_key_pwd.txt",
"interNodeTlsCrlPath" : "security/grpc/certs/",
"interNodeTlsCrlFiles" : ["server_crl.pem"],
"interNodeKmcKsfMaster" : "tools/pmt/master/ksfa",
"interNodeKmcKsfStandby" : "tools/pmt/standby/ksfb",
"ModelDeployConfig" :
{
"maxSeqLen" : 2560,
"maxInputTokenLen" : 2048,
"truncation" : false,
"ModelConfig" : [
{
"modelInstanceType" : "Standard",
"modelName" : "deepseek-r1-1b",
"modelWeightPath" : "/models/DeepSeek-R1-Distill-Qwen-1.5B",
"worldSize" : 1,
"cpuMemSize" : 5,
"npuMemSize" : -1,
"backendType" : "atb",
"trustRemoteCode" : false
}
]
},
"ScheduleConfig" :
{
"templateType" : "Standard",
"templateName" : "Standard_LLM",
"cacheBlockSize" : 128,
"maxPrefillBatchSize" : 50,
"maxPrefillTokens" : 8192,
"prefillTimeMsPerReq" : 150,
"prefillPolicyType" : 0,
"decodeTimeMsPerReq" : 50,
"decodePolicyType" : 0,
"maxBatchSize" : 200,
"maxIterTimes" : 512,
"maxPreemptCount" : 0,
"supportSelectBatch" : false,
"maxQueueDelayMicroseconds" : 5000
}
}
}