原本开发了一些代码,可以正常执行。现在机器升级了固件,执行相同的代码,程序在退出的时候,出现coredump。
```
BFD: warning: /root/anaconda3/envs/py38/lib/libgomp.so.1: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010001
BFD: warning: /root/anaconda3/envs/py38/lib/libgomp.so.1: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010002
BFD: warning: /root/anaconda3/envs/py38/lib/libquadmath.so.0: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010001
BFD: warning: /root/anaconda3/envs/py38/lib/libquadmath.so.0: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010002
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `python'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x00007f3efc867a50 in ?? ()
[Current thread is 1 (Thread 0x7f3fff3bf740 (LWP 3592770))]
(gdb) bt
#0 0x00007f3efc867a50 in ?? ()
#1 0x00007f3f3cd53546 in gert::TopologicalResourceGuard::~TopologicalResourceGuard() () from /usr/local/Ascend/ascend-toolkit/latest/lib64/libgert.so
#2 0x00007f3f3cd938d0 in std::_Rb_tree<void*, std::pair<void* const, std::unique_ptr<gert::ModelV2Executor, std::default_delete<gert::ModelV2Executor> > >, std::_Select1st<std::pair<void* const, std::unique_ptr<gert::ModelV2Executor, std::default_delete<gert::ModelV2Executor> > > >, std::less<void*>, std::allocator<std::pair<void* const, std::unique_ptr<gert::ModelV2Executor, std::default_delete<gert::ModelV2Executor> > > > >::_M_erase(std::_Rb_tree_node<std::pair<void* const, std::unique_ptr<gert::ModelV2Executor, std::default_delete<gert::ModelV2Executor> > > >*) ()
from /usr/local/Ascend/ascend-toolkit/latest/lib64/libgert.so
#3 0x00007f3f3cd90e75 in gert::StreamExecutor::~StreamExecutor() () from /usr/local/Ascend/ascend-toolkit/latest/lib64/libgert.so
#4 0x00007f3f3eb82632 in std::_Sp_counted_ptr<gert::StreamExecutor*, (__gnu_cxx::_Lock_policy)2>::_M_dispose() () from /usr/local/Ascend/ascend-toolkit/latest/lib64/libascendcl.so
#5 0x00007f3f3eb83f42 in acl::OpModelCache::CleanCachedModels() () from /usr/local/Ascend/ascend-toolkit/latest/lib64/libascendcl.so
#6 0x00007f3f3eb402ed in acl::AclResourceManager::~AclResourceManager() () from /usr/local/Ascend/ascend-toolkit/latest/lib64/libascendcl.so
#7 0x00007f3ffe431031 in __run_exit_handlers (status=0, listp=0x7f3ffe7d9718 <__exit_funcs>, run_list_atexit=run_list_atexit@entry=true, run_dtors=run_dtors@entry=true) at exit.c:108
#8 0x00007f3ffe43112a in __GI_exit (status=<optimized out>) at exit.c:139
#9 0x0000000000453eec in Py_Exit (sts=0) at /usr/local/src/conda/python-3.8.18/Programs/main.c:2295
#10 0x0000000000453e35 in handle_system_exit () at /usr/local/src/conda/python-3.8.18/Objects/clinic/marshal.c.h:684
#11 0x0000000000453af6 in _PyErr_PrintEx (tstate=0x1f13560, set_sys_last_vars=1) at /usr/local/src/conda/python-3.8.18/Objects/clinic/marshal.c.h:694
#12 0x0000000000453ace in PyErr_PrintEx (set_sys_last_vars=<optimized out>) at /usr/local/src/conda/python-3.8.18/Objects/clinic/marshal.c.h:789
#13 0x0000000000453ac3 in PyErr_Print () at /usr/local/src/conda/python-3.8.18/Objects/clinic/marshal.c.h:795
#14 0x00000000004c0ffc in PyRun_InteractiveLoopFlags (fp=fp@entry=0x7f3ffe7d9a00 <_IO_2_1_stdin_>, filename_str=filename_str@entry=0x60c18c "<stdin>", flags=flags@entry=0x7ffc821cee68)
at /usr/local/src/conda/python-3.8.18/Objects/clinic/marshal.c.h:139
#15 0x000000000045bcab in PyRun_AnyFileExFlags (fp=0x7f3ffe7d9a00 <_IO_2_1_stdin_>, filename=filename@entry=0x60c18c "<stdin>", closeit=closeit@entry=0, flags=flags@entry=0x7ffc821cee68)
at /usr/local/src/conda/python-3.8.18/Objects/clinic/marshal.c.h:84
#16 0x000000000045000c in pymain_run_stdin (cf=0x7ffc821cee68, config=0x1f125e0) at /croot/python-split_1694437953337/work/build-static/python.c:530
#17 pymain_run_python (exitcode=0x7ffc821cee60) at /croot/python-split_1694437953337/work/build-static/python.c:619
#18 Py_RunMain () at /croot/python-split_1694437953337/work/build-static/python.c:695
#19 0x0000000000579f05 in pymain_main () at /croot/python-split_1694437953337/work/build-static/python.c:1103
#20 0x0000000000579e89 in Py_BytesMain (argc=<optimized out>, argv=<optimized out>) at /croot/python-split_1694437953337/work/build-static/python.c:1127
#21 0x00007f3ffe40fc87 in __libc_start_main (main=0x579e40 <main>, argc=1, argv=0x7ffc821cf068, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffc821cf058)
at ../csu/libc-start.c:310
#22 0x0000000000579d3d in _start () at /usr/local/src/conda/python-3.8.18/Objects/clinic/import.c.h:521
(gdb) i thread
Id Target Id Frame
* 1 Thread 0x7f3fff3bf740 (LWP 3592770) 0x00007f3efc867a50 in ?? ()
2 Thread 0x7f3f354b8700 (LWP 3592950) 0x00007f3ffe4d2680 in __GI___nanosleep (requested_time=requested_time@entry=0x7f3f354b7e90, remaining=remaining@entry=0x0)
at ../sysdeps/unix/sysv/linux/nanosleep.c:28
3 Thread 0x7f3f08196700 (LWP 3593033) 0x00007f3ffef947c6 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x61b54e0) at ../sysdeps/unix/sysv/linux/futex-internal.h:205
4 Thread 0x7f3f08135700 (LWP 3593034) 0x00007f3ffe504217 in ioctl () at ../sysdeps/unix/syscall-template.S:78
```
运行阶段没有问题,退出时候崩掉了。
测试代码:
>>> x = torch.ones([2,2]).to('cuda')
>>> y = x
>>> torch.add(x, y)
fallback to cpu, name=aten::masked_select
tensor([[2., 2.],
[2., 2.]], device='cuda')
>>>
>>>
>>> del x
>>> del y
>>>
>>> torch.cuda.empty_cache()
>>>
>>> exit()
Segmentation fault (core dumped)
原本开发了一些代码,可以正常执行。现在机器升级了固件,执行相同的代码,程序在退出的时候,出现coredump。
```
BFD: warning: /root/anaconda3/envs/py38/lib/libgomp.so.1: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010001
BFD: warning: /root/anaconda3/envs/py38/lib/libgomp.so.1: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010002
BFD: warning: /root/anaconda3/envs/py38/lib/libquadmath.so.0: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010001
BFD: warning: /root/anaconda3/envs/py38/lib/libquadmath.so.0: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010002
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `python'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x00007f3efc867a50 in ?? ()
[Current thread is 1 (Thread 0x7f3fff3bf740 (LWP 3592770))]
(gdb) bt
#0 0x00007f3efc867a50 in ?? ()
#1 0x00007f3f3cd53546 in gert::TopologicalResourceGuard::~TopologicalResourceGuard() () from /usr/local/Ascend/ascend-toolkit/latest/lib64/libgert.so
#2 0x00007f3f3cd938d0 in std::_Rb_tree<void*, std::pair<void* const, std::unique_ptr<gert::ModelV2Executor, std::default_delete<gert::ModelV2Executor> > >, std::_Select1st<std::pair<void* const, std::unique_ptr<gert::ModelV2Executor, std::default_delete<gert::ModelV2Executor> > > >, std::less<void*>, std::allocator<std::pair<void* const, std::unique_ptr<gert::ModelV2Executor, std::default_delete<gert::ModelV2Executor> > > > >::_M_erase(std::_Rb_tree_node<std::pair<void* const, std::unique_ptr<gert::ModelV2Executor, std::default_delete<gert::ModelV2Executor> > > >*) ()
from /usr/local/Ascend/ascend-toolkit/latest/lib64/libgert.so
#3 0x00007f3f3cd90e75 in gert::StreamExecutor::~StreamExecutor() () from /usr/local/Ascend/ascend-toolkit/latest/lib64/libgert.so
#4 0x00007f3f3eb82632 in std::_Sp_counted_ptr<gert::StreamExecutor*, (__gnu_cxx::_Lock_policy)2>::_M_dispose() () from /usr/local/Ascend/ascend-toolkit/latest/lib64/libascendcl.so
#5 0x00007f3f3eb83f42 in acl::OpModelCache::CleanCachedModels() () from /usr/local/Ascend/ascend-toolkit/latest/lib64/libascendcl.so
#6 0x00007f3f3eb402ed in acl::AclResourceManager::~AclResourceManager() () from /usr/local/Ascend/ascend-toolkit/latest/lib64/libascendcl.so
#7 0x00007f3ffe431031 in __run_exit_handlers (status=0, listp=0x7f3ffe7d9718 <__exit_funcs>, run_list_atexit=run_list_atexit@entry=true, run_dtors=run_dtors@entry=true) at exit.c:108
#8 0x00007f3ffe43112a in __GI_exit (status=<optimized out>) at exit.c:139
#9 0x0000000000453eec in Py_Exit (sts=0) at /usr/local/src/conda/python-3.8.18/Programs/main.c:2295
#10 0x0000000000453e35 in handle_system_exit () at /usr/local/src/conda/python-3.8.18/Objects/clinic/marshal.c.h:684
#11 0x0000000000453af6 in _PyErr_PrintEx (tstate=0x1f13560, set_sys_last_vars=1) at /usr/local/src/conda/python-3.8.18/Objects/clinic/marshal.c.h:694
#12 0x0000000000453ace in PyErr_PrintEx (set_sys_last_vars=<optimized out>) at /usr/local/src/conda/python-3.8.18/Objects/clinic/marshal.c.h:789
#13 0x0000000000453ac3 in PyErr_Print () at /usr/local/src/conda/python-3.8.18/Objects/clinic/marshal.c.h:795
#14 0x00000000004c0ffc in PyRun_InteractiveLoopFlags (fp=fp@entry=0x7f3ffe7d9a00 <_IO_2_1_stdin_>, filename_str=filename_str@entry=0x60c18c "<stdin>", flags=flags@entry=0x7ffc821cee68)
at /usr/local/src/conda/python-3.8.18/Objects/clinic/marshal.c.h:139
#15 0x000000000045bcab in PyRun_AnyFileExFlags (fp=0x7f3ffe7d9a00 <_IO_2_1_stdin_>, filename=filename@entry=0x60c18c "<stdin>", closeit=closeit@entry=0, flags=flags@entry=0x7ffc821cee68)
at /usr/local/src/conda/python-3.8.18/Objects/clinic/marshal.c.h:84
#16 0x000000000045000c in pymain_run_stdin (cf=0x7ffc821cee68, config=0x1f125e0) at /croot/python-split_1694437953337/work/build-static/python.c:530
#17 pymain_run_python (exitcode=0x7ffc821cee60) at /croot/python-split_1694437953337/work/build-static/python.c:619
#18 Py_RunMain () at /croot/python-split_1694437953337/work/build-static/python.c:695
#19 0x0000000000579f05 in pymain_main () at /croot/python-split_1694437953337/work/build-static/python.c:1103
#20 0x0000000000579e89 in Py_BytesMain (argc=<optimized out>, argv=<optimized out>) at /croot/python-split_1694437953337/work/build-static/python.c:1127
#21 0x00007f3ffe40fc87 in __libc_start_main (main=0x579e40 <main>, argc=1, argv=0x7ffc821cf068, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffc821cf058)
at ../csu/libc-start.c:310
#22 0x0000000000579d3d in _start () at /usr/local/src/conda/python-3.8.18/Objects/clinic/import.c.h:521
(gdb) i thread
Id Target Id Frame
* 1 Thread 0x7f3fff3bf740 (LWP 3592770) 0x00007f3efc867a50 in ?? ()
2 Thread 0x7f3f354b8700 (LWP 3592950) 0x00007f3ffe4d2680 in __GI___nanosleep (requested_time=requested_time@entry=0x7f3f354b7e90, remaining=remaining@entry=0x0)
at ../sysdeps/unix/sysv/linux/nanosleep.c:28
3 Thread 0x7f3f08196700 (LWP 3593033) 0x00007f3ffef947c6 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x61b54e0) at ../sysdeps/unix/sysv/linux/futex-internal.h:205
4 Thread 0x7f3f08135700 (LWP 3593034) 0x00007f3ffe504217 in ioctl () at ../sysdeps/unix/syscall-template.S:78
```
运行阶段没有问题,退出时候崩掉了。
测试代码:
>>> x = torch.ones([2,2]).to('cuda')
>>> y = x
>>> torch.add(x, y)
fallback to cpu, name=aten::masked_select
tensor([[2., 2.],
[2., 2.]], device='cuda')
>>>
>>>
>>> del x
>>> del y
>>>
>>> torch.cuda.empty_cache()
>>>
>>> exit()
Segmentation fault (core dumped)