在昇腾上使用torch进行多线程会报100002ACL初始化重复错误
收藏回复举报
在昇腾上使用torch进行多线程会报100002ACL初始化重复错误
t('forum.solved') 已解决
发表于2024-05-14 11:20:00
0 查看

请问有torch多线程的例子吗?已知torch在昇腾上不支持DP,只支持DDP。

我的主程序如下:

wd.py

#-*- coding: UTF-8 -*- 
import torch_npu
import torchvision_npu
from torch_npu.contrib import transfer_to_npu

from fun_a import FUNA
from fun_b import FUNB


def run():
    
    thread_num = 10

    detect_thread_list = []
    
    funa_thread = FUNA(thread_num, thread_num)
    detect_thread_list.append(funa_thread)

    funb_thread = FUNB(thread_num, thread_num+2)
    detect_thread_list.append(funb_thread)

    # start thread    
    for t in detect_thread_list:
        t.start()
    t.join()
  
if __name__ == '__main__':
    run()

fun_a.py

# -*-coding:utf-8 -*-
import os
import threading
import torch
class FUNA(threading.Thread):
    def __init__(self, thread_num, thread_id):
        threading.Thread.__init__(self)
        self.id = thread_id
        self.thread_num = thread_num   
    def run(self):
        self._server_main()
    def _server_main(self):
        a=torch.rand(3,4)
        print("aaaaaaaaaa:",a)
        a=a.cuda()
        print("aaaaaaaaaa+a:",a+a)

fun_b.py

# -*-coding:utf-8 -*-
import os
import threading
import torch
class FUNB(threading.Thread):
    def __init__(self, thread_num, thread_id):
        threading.Thread.__init__(self)
        self.id = thread_id
        self.thread_num = thread_num
    def run(self):
        self._server_main()
    def _server_main(self):
        a=torch.rand(3,4)
        print("bbbbbbbb:",a)
        a=a.cuda()
        print("bbbbbbb+b:",a+a)

执行python wd.py,会报100002ACL初始化错误。报错行在第二个线程的a=a.cuda(),设置device错误。日志如下:

  warnings.warn(msg, ImportWarning) 

Exception in thread Thread-2: 

Traceback (most recent call last): 

  File "/usr/local/python3.7.5/lib/python3.7/threading.py", line 926, in _bootstrap_inner 

    self.run() 

  File "/home/xxxx/fun_b.py", line 11, in run 

    self._server_main() 

  File "/home/xxxx/fun_b.py", line 15, in _server_main 

    a=a.cuda() 

  File "/usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/utils/device_guard.py", line 38, in wrapper 

    return func(*args, **kwargs) 

  File "/usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/utils/tensor_methods.py", line 66, in _npu 

    return torch_npu._C.npu(self, *args, **kwargs) 

RuntimeError: Initialize:/usr1/workspace/FPTA_Daily_open_pytorchv1.11.0-5.0.rc1/CODE/torch_npu/csrc/core/npu/sys_ctrl/npu_sys_ctrl.cpp:103 NPU error, error code is 100002 

EE8888: Inner Error! 

        ctx is NULL![FUNC:GetDevErrMsg][FILE:api_impl.cc][LINE:3680] 

        rtGetDevMsg execute failed, reason=[context pointer null][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:49] 

 

aaaaaaaaaa: bbbbbbbb: tensor([[0.1819, 0.5542, 0.4418, 0.7263], 

        [0.5195, 0.3939, 0.5962, 0.8784], 

        [0.6600, 0.5914, 0.1660, 0.4286]]) 

tensor([[0.6525, 0.5055, 0.6912, 0.4498], 

        [0.7493, 0.0386, 0.5924, 0.0423], 

        [0.9474, 0.1643, 0.5311, 0.1686]]) 

terminate called after throwing an instance of 'c10::Error' 

  what():  current_device:/usr1/workspace/FPTA_Daily_open_pytorchv1.11.0-5.0.rc1/CODE/torch_npu/csrc/core/npu/NPUFunctions.h:38 NPU error, error code is 107002 

EE8888: Inner Error! 

        ctx is NULL![FUNC:GetDevErrMsg][FILE:api_impl.cc][LINE:3680] 

        rtGetDevMsg execute failed, reason=[context pointer null][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:49] 

 

EE8888: Inner Error! 

        ctx is NULL![FUNC:GetDevErrMsg][FILE:api_impl.cc][LINE:3680] 

        rtGetDevMsg execute failed, reason=[context pointer null][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:49] 

 

EE8888: Inner Error! 

        ctx is NULL![FUNC:GetDevErrMsg][FILE:api_impl.cc][LINE:3680] 

        rtGetDevMsg execute failed, reason=[context pointer null][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:49] 

 

Exception raised from current_device at /usr1/workspace/FPTA_Daily_open_pytorchv1.11.0-5.0.rc1/CODE/torch_npu/csrc/core/npu/NPUFunctions.h:38 (most recent call first): 

frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x8c (0xffff8ecb6114 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch/lib/libc10.so) 

frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0xa0 (0xffff8ecb2418 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch/lib/libc10.so) 

frame #2: c10_npu::getCurrentNPUStream(signed char) + 0x1b0 (0xfffeed84d7d0 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/lib/libtorch_npu.so) 

frame #3: <unknown function> + 0x869a5c (0xfffeed85ba5c in /usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/lib/libtorch_npu.so) 

frame #4: c10_npu::NpuSysCtrl::Finalize() + 0xf8 (0xfffeed85c200 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/lib/libtorch_npu.so) 

frame #5: THPModule_npu_shutdown(_object*) + 0xc8 (0xfffeedc23490 in /usr/local/python3.7.5/lib/python3.7/site-packages/torch_npu/_C.cpython-37m-aarch64-linux-gnu.so) 

<omitting python frames> 

 

Process ForkServerProcess-2: 

Traceback (most recent call last): 

  File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap 

    self.run() 

  File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/process.py", line 99, in run 

    self._target(*self._args, **self._kwargs) 

  File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 61, in wrapper 

    raise exp 

  File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 58, in wrapper 

    func(*args, **kwargs) 

  File "/usr/local/Ascend/ascend-toolkit/latest/python/site-packages/tbe/common/repository_manager/route.py", line 275, in task_distribute 

    key, func_name, detail = resource_proxy[TASK_QUEUE].get() 

  File "<string>", line 2, in get 

  File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/managers.py", line 819, in _callmethod 

    kind, result = conn.recv() 

  File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/connection.py", line 250, in recv 

    buf = self._recv_bytes() 

  File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/connection.py", line 407, in _recv_bytes 

    buf = self._recv(4) 

  File "/usr/local/python3.7.5/lib/python3.7/multiprocessing/connection.py", line 383, in _recv 

    raise EOFError 

EOFError 

/usr/local/python3.7.5/lib/python3.7/multiprocessing/semaphore_tracker.py:144: UserWarning: semaphore_tracker: There appear to be 91 leaked semaphores to clean up at shutdown 

  len(cache)) 

我要发帖子