请问有torch多线程的例子吗?已知torch在昇腾上不支持DP,只支持DDP。
我的主程序如下:
wd.py
fun_a.py
fun_b.py
执行python wd.py,会报100002ACL初始化错误。报错行在第二个线程的a=a.cuda(),设置device错误。日志如下:
warnings.warn(msg, ImportWarning)
Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/local/python3.7.5/lib/python3.7/threading.py", line 926, in _bootstrap_inner
self.run()
File "/home/xxxx/fun_b.py", line 11, in run
self._server_main()
File "/home/xxxx/fun_b.py", line 15, in _server_main
a=a.cuda()
File "/usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/utils/device_guard.py", line 38, in wrapper
return func(*args, **kwargs)
File "/usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/utils/tensor_methods.py", line 66, in _npu
return torch_npu._C.npu(self, *args, **kwargs)
RuntimeError: Initialize:/usr1/workspace/FPTA_Daily_open_pytorchv1.11.0-5.0.rc1/CODE/torch_npu/csrc/core/npu/sys_ctrl/npu_sys_ctrl.cpp:103 NPU error, error code is 100002
EE8888: Inner Error!
ctx is NULL![FUNC:GetDevErrMsg][FILE:api_impl.cc][LINE:3680]
rtGetDevMsg execute failed, reason=[context pointer null][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:49]
aaaaaaaaaa: bbbbbbbb: tensor([[0.1819, 0.5542, 0.4418, 0.7263],
[0.5195, 0.3939, 0.5962, 0.8784],
[0.6600, 0.5914, 0.1660, 0.4286]])
tensor([[0.6525, 0.5055, 0.6912, 0.4498],
[0.7493, 0.0386, 0.5924, 0.0423],
[0.9474, 0.1643, 0.5311, 0.1686]])
terminate called after throwing an instance of 'c10::Error'
what(): current_device:/usr1/workspace/FPTA_Daily_open_pytorchv1.11.0-5.0.rc1/CODE/torch_npu/csrc/core/npu/NPUFunctions.h:38 NPU error, error code is 107002
EE8888: Inner Error!
ctx is NULL![FUNC:GetDevErrMsg][FILE:api_impl.cc][LINE:3680]
rtGetDevMsg execute failed, reason=[context pointer null][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:49]
EE8888: Inner Error!
ctx is NULL![FUNC:GetDevErrMsg][FILE:api_impl.cc][LINE:3680]
rtGetDevMsg execute failed, reason=[context pointer null][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:49]
EE8888: Inner Error!
ctx is NULL![FUNC:GetDevErrMsg][FILE:api_impl.cc][LINE:3680]
rtGetDevMsg execute failed, reason=[context pointer null][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:49]
Exception raised from current_device at /usr1/workspace/FPTA_Daily_open_pytorchv1.11.0-5.0.rc1/CODE/torch_npu/csrc/core/npu/NPUFunctions.h:38 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x8c (0xffff8ecb6114 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0xa0 (0xffff8ecb2418 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #2: c10_npu::getCurrentNPUStream(signed char) + 0x1b0 (0xfffeed84d7d0 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/lib/libtorch_npu.so)
frame #3: <unknown function> + 0x869a5c (0xfffeed85ba5c in /usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/lib/libtorch_npu.so)
frame #4: c10_npu::NpuSysCtrl::Finalize() + 0xf8 (0xfffeed85c200 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/lib/libtorch_npu.so)
frame #5: THPModule_npu_shutdown(_object*) + 0xc8 (0xfffeedc23490 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/_C.cpython-37m-aarch64-linux-gnu.so)
<omitting python frames>
Process ForkServerProcess-2:
Traceback (most recent call last):
File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 61, in wrapper
raise exp
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 58, in wrapper
func(*args, **kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 275, in task_distribute
key, func_name, detail = resource_proxy[TASK_QUEUE].get()
File "<string>", line 2, in get
File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/managers.py", line 819, in _callmethod
kind, result = conn.recv()
File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/connection.py", line 250, in recv
buf = self._recv_bytes()
File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/connection.py", line 407, in _recv_bytes
buf = self._recv(4)
File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/connection.py", line 383, in _recv
raise EOFError
EOFError
/usr/local/python3.7.5/lib/python3.7/multiprocessing/semaphore_tracker.py:144: UserWarning: semaphore_tracker: There appear to be 91 leaked semaphores to clean up at shutdown
len(cache))
请问有torch多线程的例子吗?已知torch在昇腾上不支持DP,只支持DDP。
我的主程序如下:
wd.py
#-*- coding: UTF-8 -*- import torch_npu import torchvision_npu from torch_npu.contrib import transfer_to_npu from fun_a import FUNA from fun_b import FUNB def run(): thread_num = 10 detect_thread_list = [] funa_thread = FUNA(thread_num, thread_num) detect_thread_list.append(funa_thread) funb_thread = FUNB(thread_num, thread_num+2) detect_thread_list.append(funb_thread) # start thread for t in detect_thread_list: t.start() t.join() if __name__ == '__main__': run()fun_a.py
# -*-coding:utf-8 -*- import os import threading import torch class FUNA(threading.Thread): def __init__(self, thread_num, thread_id): threading.Thread.__init__(self) self.id = thread_id self.thread_num = thread_num def run(self): self._server_main() def _server_main(self): a=torch.rand(3,4) print("aaaaaaaaaa:",a) a=a.cuda() print("aaaaaaaaaa+a:",a+a)fun_b.py
# -*-coding:utf-8 -*- import os import threading import torch class FUNB(threading.Thread): def __init__(self, thread_num, thread_id): threading.Thread.__init__(self) self.id = thread_id self.thread_num = thread_num def run(self): self._server_main() def _server_main(self): a=torch.rand(3,4) print("bbbbbbbb:",a) a=a.cuda() print("bbbbbbb+b:",a+a)执行python wd.py,会报100002ACL初始化错误。报错行在第二个线程的a=a.cuda(),设置device错误。日志如下:
warnings.warn(msg, ImportWarning)
Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/local/python3.7.5/lib/python3.7/threading.py", line 926, in _bootstrap_inner
self.run()
File "/home/xxxx/fun_b.py", line 11, in run
self._server_main()
File "/home/xxxx/fun_b.py", line 15, in _server_main
a=a.cuda()
File "/usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/utils/device_guard.py", line 38, in wrapper
return func(*args, **kwargs)
File "/usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/utils/tensor_methods.py", line 66, in _npu
return torch_npu._C.npu(self, *args, **kwargs)
RuntimeError: Initialize:/usr1/workspace/FPTA_Daily_open_pytorchv1.11.0-5.0.rc1/CODE/torch_npu/csrc/core/npu/sys_ctrl/npu_sys_ctrl.cpp:103 NPU error, error code is 100002
EE8888: Inner Error!
ctx is NULL![FUNC:GetDevErrMsg][FILE:api_impl.cc][LINE:3680]
rtGetDevMsg execute failed, reason=[context pointer null][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:49]
aaaaaaaaaa: bbbbbbbb: tensor([[0.1819, 0.5542, 0.4418, 0.7263],
[0.5195, 0.3939, 0.5962, 0.8784],
[0.6600, 0.5914, 0.1660, 0.4286]])
tensor([[0.6525, 0.5055, 0.6912, 0.4498],
[0.7493, 0.0386, 0.5924, 0.0423],
[0.9474, 0.1643, 0.5311, 0.1686]])
terminate called after throwing an instance of 'c10::Error'
what(): current_device:/usr1/workspace/FPTA_Daily_open_pytorchv1.11.0-5.0.rc1/CODE/torch_npu/csrc/core/npu/NPUFunctions.h:38 NPU error, error code is 107002
EE8888: Inner Error!
ctx is NULL![FUNC:GetDevErrMsg][FILE:api_impl.cc][LINE:3680]
rtGetDevMsg execute failed, reason=[context pointer null][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:49]
EE8888: Inner Error!
ctx is NULL![FUNC:GetDevErrMsg][FILE:api_impl.cc][LINE:3680]
rtGetDevMsg execute failed, reason=[context pointer null][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:49]
EE8888: Inner Error!
ctx is NULL![FUNC:GetDevErrMsg][FILE:api_impl.cc][LINE:3680]
rtGetDevMsg execute failed, reason=[context pointer null][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:49]
Exception raised from current_device at /usr1/workspace/FPTA_Daily_open_pytorchv1.11.0-5.0.rc1/CODE/torch_npu/csrc/core/npu/NPUFunctions.h:38 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x8c (0xffff8ecb6114 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0xa0 (0xffff8ecb2418 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #2: c10_npu::getCurrentNPUStream(signed char) + 0x1b0 (0xfffeed84d7d0 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/lib/libtorch_npu.so)
frame #3: <unknown function> + 0x869a5c (0xfffeed85ba5c in /usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/lib/libtorch_npu.so)
frame #4: c10_npu::NpuSysCtrl::Finalize() + 0xf8 (0xfffeed85c200 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/lib/libtorch_npu.so)
frame #5: THPModule_npu_shutdown(_object*) + 0xc8 (0xfffeedc23490 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/_C.cpython-37m-aarch64-linux-gnu.so)
<omitting python frames>
Process ForkServerProcess-2:
Traceback (most recent call last):
File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 61, in wrapper
raise exp
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 58, in wrapper
func(*args, **kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 275, in task_distribute
key, func_name, detail = resource_proxy[TASK_QUEUE].get()
File "<string>", line 2, in get
File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/managers.py", line 819, in _callmethod
kind, result = conn.recv()
File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/connection.py", line 250, in recv
buf = self._recv_bytes()
File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/connection.py", line 407, in _recv_bytes
buf = self._recv(4)
File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/connection.py", line 383, in _recv
raise EOFError
EOFError
/usr/local/python3.7.5/lib/python3.7/multiprocessing/semaphore_tracker.py:144: UserWarning: semaphore_tracker: There appear to be 91 leaked semaphores to clean up at shutdown
len(cache))