---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
/tmp/ipykernel_617280/3533520499.py in <cell line: 26>()
     24     ssd_eval(mindrecord_file, "./ssd-60_9.ckpt", anno_json)
     25 
---> 26 eval_net()

/tmp/ipykernel_617280/3533520499.py in eval_net()
     22 def eval_net():
     23     print("Start Eval!")
---> 24     ssd_eval(mindrecord_file, "./ssd-60_9.ckpt", anno_json)
     25 
     26 eval_net()

/tmp/ipykernel_617280/3533520499.py in ssd_eval(dataset_path, ckpt_path, anno_json)
     16     print("total images num: ", total)
     17     eval_param_dict = {"net": net, "dataset": ds, "anno_json": anno_json}
---> 18     mAP = apply_eval(eval_param_dict)
     19     print("\n========================================\n")
     20     print(f"mAP: {mAP}")

/tmp/ipykernel_617280/4101351943.py in apply_eval(eval_param_dict)
     23         image_shape = data['image_shape']
     24 
---> 25         output = net(Tensor(img_np))
     26 
     27         print("run22")

/usr/local/miniconda3/lib/python3.9/site-packages/mindspore/nn/cell.py in __call__(self, *args, **kwargs)
    720                                f"function, please use context.set_context to set pynative mode.")
    721             self._self_check()
--> 722             out = self.compile_and_run(*args, **kwargs)
    723             return out
    724 

/usr/local/miniconda3/lib/python3.9/site-packages/mindspore/nn/cell.py in compile_and_run(self, *args, **kwargs)
   1141         self.add_flags(ge_sync_data=False)
   1142         new_args = _get_args_for_run(self, args, kwargs, self._compile_args)
-> 1143         return _cell_graph_executor(self, *new_args, phase=self.phase)
   1144 
   1145     def auto_parallel_compile_and_run(self):

/usr/local/miniconda3/lib/python3.9/site-packages/mindspore/common/api.py in __call__(self, obj, phase, *args)
   1878         if context.get_context("precompile_only") or _is_role_sched():
   1879             return None
-> 1880         return self.run(obj, *args, phase=phase)
   1881 
   1882     def has_compiled(self, phase='predict'):

/usr/local/miniconda3/lib/python3.9/site-packages/mindspore/common/api.py in run(self, obj, phase, *args)
   1929         phase_real = phase + '.' + str(obj.create_time) + '.' + str(id(obj)) + '.' + obj.arguments_key
   1930         if self.has_compiled(phase_real):
-> 1931             return self._exec_pip(obj, *args, phase=phase_real)
   1932         raise KeyError('{} graph is not exist.'.format(phase_real))
   1933 

/usr/local/miniconda3/lib/python3.9/site-packages/mindspore/common/api.py in wrapper(*arg, **kwargs)
    186     @wraps(fn)
    187     def wrapper(*arg, **kwargs):
--> 188         results = fn(*arg, **kwargs)
    189         return _convert_python_data(results)
    190 

/usr/local/miniconda3/lib/python3.9/site-packages/mindspore/common/api.py in _exec_pip(self, obj, phase, *args)
   1909         fn = obj.construct
   1910         obj.__parse_method__ = fn.__name__
-> 1911         return self._graph_executor(args, phase)
   1912 
   1913     def run(self, obj, *args, phase='predict'):

RuntimeError: Sync stream failed:Ascend_0

----------------------------------------------------
- Ascend Error Message:
----------------------------------------------------
E39999: Inner Error!
E39999: [PID: 617280] 2025-01-23-10:07:32.203.403 The error from device(chipId:0, dieId:0), serial number is 4, an exception occurred during AICPU execution, stream_id:8, task_id:11, errcode:5, msg:aicpu execute failed..[FUNC:ProcessStarsAicpuErrorInfo][FILE:device_error_proc.cc][LINE:1479][THREAD:617951]
        TraceBack (most recent call last):
       Kernel task happen error, retCode=0x2a, [aicpu exception].[FUNC:PreCheckTaskErr][FILE:davinci_kernel_task.cc][LINE:1356][THREAD:617951]
       AICPU Kernel task happen error, retCode=0x2a.[FUNC:GetError][FILE:stream.cc][LINE:1124][THREAD:617951]
       Aicpu kernel execute failed, device_id=0, stream_id=8, task_id=11, errorCode=2a.[FUNC:PrintAicpuErrorInfo][FILE:davinci_kernel_task.cc][LINE:1120][THREAD:617951]
       Aicpu kernel execute failed, device_id=0, stream_id=8, task_id=11, flip_num=0, kernel_type=1fault so_name=, fault kernel_name=, fault op_name=, extend_info=.[FUNC:GetError][FILE:stream.cc][LINE:1124][THREAD:617951]
       rtStreamSynchronizeWithTimeout execute failed, reason=[aicpu exception][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:53][THREAD:617951]
       synchronize stream failed, runtime result = 507018[FUNC:ReportCallError][FILE:log_inner.cpp][LINE:161][THREAD:617951]

(Please search "CANN Common Error Analysis" at https://www.mindspore.cn for error code description)

----------------------------------------------------
- C++ Call Stack: (For framework developers)
----------------------------------------------------
mindspore/ccsrc/runtime/graph_scheduler/graph_scheduler.cc:1073 Run