conda create -n py310 python=3.10
conda activate py310
apt update

pytest ./ascend/examples/pytest_ut/test_add.py

FileNotFoundError: [Errno 2] No such file or directory: 'npuc'

============================= test session starts ==============================
platform linux -- Python 3.10.16, pytest-8.3.5, pluggy-1.6.0
rootdir: /data/home/2200012812/samples/triton-ascend
collected 3 items

ascend/examples/pytest_ut/test_add.py FFF                                [100%]

=================================== FAILURES ===================================
____________________________ test_case[param_list0] ____________________________

src = <triton.triton_patch.compiler.compiler.ASTSource object at 0xffffa4771630>
target = GPUTarget(backend='npu', arch='Ascend910B3', warp_size=0)
options = NPUOptions(debug=False, sanitize_overflow=True, llvm_version=15, kernel_name='triton_', cluster_dims=(1, 1, 1), num_wa...ions=('ieee', 'hf32'), enable_npu_compile=True, max_num_imprecise_acc_default=None, extern_libs=None, multibuffer=True)

    def compile(src, target=None, options=None):
        from triton.backends.compiler import GPUTarget
        from triton.runtime.cache import get_cache_manager, get_dump_manager, get_override_manager
        from triton.runtime.driver import driver
        from .errors import MLIRCompilationError
        if target is None:
            target = driver.active.get_current_target()
        assert isinstance(target, GPUTarget), "target must be of GPUTarget type"
        backend = make_backend(target)
        ir_source = not isinstance(src, ASTSource)
        # create backend
        if ir_source:
            assert isinstance(src, str), "source must be either AST or a filepath"
            src = IRSource(src)
        extra_options = src.parse_options()
        options = backend.parse_options(dict(options or dict(), **extra_options))
        # create cache manager
        env_vars = get_cache_invalidating_env_vars()
        key = f"{triton_key()}-{src.hash()}-{backend.hash()}-{options.hash()}-{str(sorted(env_vars.items()))}"
        hash = hashlib.sha256(key.encode("utf-8")).hexdigest()
        fn_cache_manager = get_cache_manager(hash)
        # For dumping/overriding only hash the source as we want it to be independent of triton
        # core changes to make it easier to track kernels by hash.
        enable_override = os.environ.get("TRITON_KERNEL_OVERRIDE", "0") == "1"
        enable_ir_dump = os.environ.get("TRITON_KERNEL_DUMP", "0") == "1"
        fn_override_manager = get_override_manager(src.hash()) if enable_override else None
        fn_dump_manager = get_dump_manager(src.hash()) if enable_ir_dump else None
        # Pre-truncate the file name here to avoid hitting the 255 character limit on common platforms.
        # The final file name in the cache will have a format of f"{filename}.{ext}.tmp.pid_{pid}_{uuid}".
        # A PID string can be 5-character long. A UUID string has typically 36 characters. Let's truncate
        # the file name to 150 characters to be safe.
        file_name = src.name[:150]
        metadata_filename = f"{file_name}.json"
        metadata_group = fn_cache_manager.get_group(metadata_filename) or {}
        metadata_path = metadata_group.get(metadata_filename)
        always_compile = os.environ.get("TRITON_ALWAYS_COMPILE", "0") == "1"
        if not always_compile and metadata_path is not None:
            # cache hit!
            metadata = json.loads(Path(metadata_path).read_text())
            return CompiledKernel(src, metadata_group, hash)
        compile_speed_opt = os.getenv("TRITON_ASCEND_COMPILE_SPEED_OPT", 'false').lower() in ('true', '1')
        if (compile_speed_opt):
            ttir_path = f"{file_name}.ttir"
            if (metadata_path is None) and (fn_cache_manager.has_file(ttir_path)):
                # Already compile once but failed. So directly return
                raise Exception("already failed once")
        # initialize metadata
        metadata = {
            "hash": hash,
            "target": target,
            **options.__dict__,
            **env_vars,
        }
        # run compilation pipeline  and populate metadata
        stages = dict()
        backend.add_stages(stages, options)
        first_stage = list(stages.keys()).index(src.ext)
        # when the source is an IR file, don't apply the passes related to this stage. This makes it easier to write IR level tests.
        if ir_source:
            first_stage += 1
        context = ir.context()
        ir.load_dialects(context)
        backend.load_dialects(context)
        codegen_fns = backend.get_codegen_implementation()
        module_map = backend.get_module_map()
        try:
            module = src.make_ir(options, codegen_fns, module_map, context)
        except Exception as e:
            filter_traceback(e)
            raise
        use_ir_loc = os.environ.get("USE_IR_LOC", None)
        for ext, compile_ir in list(stages.items())[first_stage:]:
            try:
>               next_module = compile_ir(module, metadata)

/root/software/miniconda3/envs/py310/lib/python3.10/site-packages/triton/triton_patch/compiler/compiler.py:284: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/root/software/miniconda3/envs/py310/lib/python3.10/site-packages/triton/backends/huawei/compiler.py:356: in <lambda>
    stages["npubin"] = lambda src, metadata: linalg_to_bin_enable_npu_compile(src, metadata, options)
/root/software/miniconda3/envs/py310/lib/python3.10/site-packages/triton/backends/huawei/compiler.py:194: in linalg_to_bin_enable_npu_compile
    ret = subprocess.run(cmd_list, capture_output=True, check=True)
/root/software/miniconda3/envs/py310/lib/python3.10/subprocess.py:503: in run
    with Popen(*popenargs, **kwargs) as process:
/root/software/miniconda3/envs/py310/lib/python3.10/subprocess.py:971: in __init__
    self._execute_child(args, executable, preexec_fn, close_fds,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <Popen: returncode: 255 args: ['npuc', '/tmp/tmp7ao7iau3/kernel.ttadapter.ml...>
args = ['npuc', '/tmp/tmp7ao7iau3/kernel.ttadapter.mlir', '--enable-auto-multi-buffer=True', '-o', '/tmp/tmp7ao7iau3/kernel']
executable = b'npuc', preexec_fn = None, close_fds = True, pass_fds = ()
cwd = None, env = None, startupinfo = None, creationflags = 0, shell = False
p2cread = -1, p2cwrite = -1, c2pread = 91, c2pwrite = 92, errread = 93
errwrite = 94, restore_signals = True, gid = None, gids = None, uid = None
umask = -1, start_new_session = False

    def _execute_child(self, args, executable, preexec_fn, close_fds,
                       pass_fds, cwd, env,
                       startupinfo, creationflags, shell,
                       p2cread, p2cwrite,
                       c2pread, c2pwrite,
                       errread, errwrite,
                       restore_signals,
                       gid, gids, uid, umask,
                       start_new_session):
        """Execute program (POSIX version)"""
    
        if isinstance(args, (str, bytes)):
            args = [args]
        elif isinstance(args, os.PathLike):
            if shell:
                raise TypeError('path-like args is not allowed when '
                                'shell is true')
            args = [args]
        else:
            args = list(args)
    
        if shell:
            # On Android the default shell is at '/system/bin/sh'.
            unix_shell = ('/system/bin/sh' if
                      hasattr(sys, 'getandroidapilevel') else '/bin/sh')
            args = [unix_shell, "-c"] + args
            if executable:
                args[0] = executable
    
        if executable is None:
            executable = args[0]
    
        sys.audit("subprocess.Popen", executable, args, cwd, env)
    
        if (_USE_POSIX_SPAWN
                and os.path.dirname(executable)
                and preexec_fn is None
                and not close_fds
                and not pass_fds
                and cwd is None
                and (p2cread == -1 or p2cread > 2)
                and (c2pwrite == -1 or c2pwrite > 2)
                and (errwrite == -1 or errwrite > 2)
                and not start_new_session
                and gid is None
                and gids is None
                and uid is None
                and umask < 0):
            self._posix_spawn(args, executable, env, restore_signals,
                              p2cread, p2cwrite,
                              c2pread, c2pwrite,
                              errread, errwrite)
            return
    
        orig_executable = executable
    
        # For transferring possible exec failure from child to parent.
        # Data format: "exception name:hex errno:description"
        # Pickle is not used; it is complex and involves memory allocation.
        errpipe_read, errpipe_write = os.pipe()
        # errpipe_write must not be in the standard io 0, 1, or 2 fd range.
        low_fds_to_close = []
        while errpipe_write < 3:
            low_fds_to_close.append(errpipe_write)
            errpipe_write = os.dup(errpipe_write)
        for low_fd in low_fds_to_close:
            os.close(low_fd)
        try:
            try:
                # We must avoid complex work that could involve
                # malloc or free in the child process to avoid
                # potential deadlocks, thus we do all this here.
                # and pass it to fork_exec()
    
                if env is not None:
                    env_list = []
                    for k, v in env.items():
                        k = os.fsencode(k)
                        if b'=' in k:
                            raise ValueError("illegal environment variable name")
                        env_list.append(k + b'=' + os.fsencode(v))
                else:
                    env_list = None  # Use execv instead of execve.
                executable = os.fsencode(executable)
                if os.path.dirname(executable):
                    executable_list = (executable,)
                else:
                    # This matches the behavior of os._execvpe().
                    executable_list = tuple(
                        os.path.join(os.fsencode(dir), executable)
                        for dir in os.get_exec_path(env))
                fds_to_keep = set(pass_fds)
                fds_to_keep.add(errpipe_write)
                self.pid = _posixsubprocess.fork_exec(
                        args, executable_list,
                        close_fds, tuple(sorted(map(int, fds_to_keep))),
                        cwd, env_list,
                        p2cread, p2cwrite, c2pread, c2pwrite,
                        errread, errwrite,
                        errpipe_read, errpipe_write,
                        restore_signals, start_new_session,
                        gid, gids, uid, umask,
                        preexec_fn)
                self._child_created = True
            finally:
                # be sure the FD is closed no matter what
                os.close(errpipe_write)
    
            self._close_pipe_fds(p2cread, p2cwrite,
                                 c2pread, c2pwrite,
                                 errread, errwrite)
    
            # Wait for exec to fail or succeed; possibly raising an
            # exception (limited in size)
            errpipe_data = bytearray()
            while True:
                part = os.read(errpipe_read, 50000)
                errpipe_data += part
                if not part or len(errpipe_data) > 50000:
                    break
        finally:
            # be sure the FD is closed no matter what
            os.close(errpipe_read)
    
        if errpipe_data:
            try:
                pid, sts = os.waitpid(self.pid, 0)
                if pid == self.pid:
                    self._handle_exitstatus(sts)
                else:
                    self.returncode = sys.maxsize
            except ChildProcessError:
                pass
    
            try:
                exception_name, hex_errno, err_msg = (
                        errpipe_data.split(b':', 2))
                # The encoding here should match the encoding
                # written in by the subprocess implementations
                # like _posixsubprocess
                err_msg = err_msg.decode()
            except ValueError:
                exception_name = b'SubprocessError'
                hex_errno = b'0'
                err_msg = 'Bad exception data from child: {!r}'.format(
                              bytes(errpipe_data))
            child_exception_type = getattr(
                    builtins, exception_name.decode('ascii'),
                    SubprocessError)
            if issubclass(child_exception_type, OSError) and hex_errno:
                errno_num = int(hex_errno, 16)
                child_exec_never_called = (err_msg == "noexec")
                if child_exec_never_called:
                    err_msg = ""
                    # The error must be from chdir(cwd).
                    err_filename = cwd
                else:
                    err_filename = orig_executable
                if errno_num != 0:
                    err_msg = os.strerror(errno_num)
>               raise child_exception_type(errno_num, err_msg, err_filename)
E               FileNotFoundError: [Errno 2] No such file or directory: 'npuc'

/root/software/miniconda3/envs/py310/lib/python3.10/subprocess.py:1863: FileNotFoundError

During handling of the above exception, another exception occurred:

param_list = ['float32', (2, 4096, 8), 2, 32768, 1024]

    @pytest.mark.parametrize('param_list',
                             [
                                 ['float32', (2, 4096, 8), 2, 32768, 1024],
                                 ['float16', (2, 4096, 8), 2, 32768, 1024],
                                 ['int8', (2, 4096, 8), 2, 32768, 1024],
                             ]
                             )
    
    def test_case(param_list):
        dtype, shape, ncore, xblock, xblock_sub = param_list
        x0 = test_common.generate_tensor(shape, dtype).npu()
        x1 = test_common.generate_tensor(shape, dtype).npu()
        y_ref = torch_pointwise(x0, x1)
        y_cal = torch.zeros(shape, dtype = eval('torch.' + dtype)).npu()
>       triton_add[ncore, 1, 1](x0, x1, y_cal, xblock, xblock_sub)

ascend/examples/pytest_ut/test_add.py:41: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/root/software/miniconda3/envs/py310/lib/python3.10/site-packages/triton/runtime/jit.py:330: in <lambda>
    return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
/root/software/miniconda3/envs/py310/lib/python3.10/site-packages/triton/runtime/jit.py:623: in run
    kernel = self.compile(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

src = <triton.triton_patch.compiler.compiler.ASTSource object at 0xffffa4771630>
target = GPUTarget(backend='npu', arch='Ascend910B3', warp_size=0)
options = NPUOptions(debug=False, sanitize_overflow=True, llvm_version=15, kernel_name='triton_', cluster_dims=(1, 1, 1), num_wa...ions=('ieee', 'hf32'), enable_npu_compile=True, max_num_imprecise_acc_default=None, extern_libs=None, multibuffer=True)

    def compile(src, target=None, options=None):
        from triton.backends.compiler import GPUTarget
        from triton.runtime.cache import get_cache_manager, get_dump_manager, get_override_manager
        from triton.runtime.driver import driver
        from .errors import MLIRCompilationError
        if target is None:
            target = driver.active.get_current_target()
        assert isinstance(target, GPUTarget), "target must be of GPUTarget type"
        backend = make_backend(target)
        ir_source = not isinstance(src, ASTSource)
        # create backend
        if ir_source:
            assert isinstance(src, str), "source must be either AST or a filepath"
            src = IRSource(src)
        extra_options = src.parse_options()
        options = backend.parse_options(dict(options or dict(), **extra_options))
        # create cache manager
        env_vars = get_cache_invalidating_env_vars()
        key = f"{triton_key()}-{src.hash()}-{backend.hash()}-{options.hash()}-{str(sorted(env_vars.items()))}"
        hash = hashlib.sha256(key.encode("utf-8")).hexdigest()
        fn_cache_manager = get_cache_manager(hash)
        # For dumping/overriding only hash the source as we want it to be independent of triton
        # core changes to make it easier to track kernels by hash.
        enable_override = os.environ.get("TRITON_KERNEL_OVERRIDE", "0") == "1"
        enable_ir_dump = os.environ.get("TRITON_KERNEL_DUMP", "0") == "1"
        fn_override_manager = get_override_manager(src.hash()) if enable_override else None
        fn_dump_manager = get_dump_manager(src.hash()) if enable_ir_dump else None
        # Pre-truncate the file name here to avoid hitting the 255 character limit on common platforms.
        # The final file name in the cache will have a format of f"{filename}.{ext}.tmp.pid_{pid}_{uuid}".
        # A PID string can be 5-character long. A UUID string has typically 36 characters. Let's truncate
        # the file name to 150 characters to be safe.
        file_name = src.name[:150]
        metadata_filename = f"{file_name}.json"
        metadata_group = fn_cache_manager.get_group(metadata_filename) or {}
        metadata_path = metadata_group.get(metadata_filename)
        always_compile = os.environ.get("TRITON_ALWAYS_COMPILE", "0") == "1"
        if not always_compile and metadata_path is not None:
            # cache hit!
            metadata = json.loads(Path(metadata_path).read_text())
            return CompiledKernel(src, metadata_group, hash)
        compile_speed_opt = os.getenv("TRITON_ASCEND_COMPILE_SPEED_OPT", 'false').lower() in ('true', '1')
        if (compile_speed_opt):
            ttir_path = f"{file_name}.ttir"
            if (metadata_path is None) and (fn_cache_manager.has_file(ttir_path)):
                # Already compile once but failed. So directly return
                raise Exception("already failed once")
        # initialize metadata
        metadata = {
            "hash": hash,
            "target": target,
            **options.__dict__,
            **env_vars,
        }
        # run compilation pipeline  and populate metadata
        stages = dict()
        backend.add_stages(stages, options)
        first_stage = list(stages.keys()).index(src.ext)
        # when the source is an IR file, don't apply the passes related to this stage. This makes it easier to write IR level tests.
        if ir_source:
            first_stage += 1
        context = ir.context()
        ir.load_dialects(context)
        backend.load_dialects(context)
        codegen_fns = backend.get_codegen_implementation()
        module_map = backend.get_module_map()
        try:
            module = src.make_ir(options, codegen_fns, module_map, context)
        except Exception as e:
            filter_traceback(e)
            raise
        use_ir_loc = os.environ.get("USE_IR_LOC", None)
        for ext, compile_ir in list(stages.items())[first_stage:]:
            try:
                next_module = compile_ir(module, metadata)
            except Exception as e:
                if (ext == "ttadapter"):
                    stage_name = "ConvertTritonIRToLinalgIR"
                elif (ext == "npubin"):
                    stage_name = "ConvertLinalgRToBinary"
                else:
                    stage_name = "MLIRCompile"
>               raise MLIRCompilationError(stage_name, e.stderr.decode('utf-8'))
E               AttributeError: 'FileNotFoundError' object has no attribute 'stderr'

/root/software/miniconda3/envs/py310/lib/python3.10/site-packages/triton/triton_patch/compiler/compiler.py:292: AttributeError

 ... 帖子放不下了，故省略

=========================== short test summary info ============================
FAILED ascend/examples/pytest_ut/test_add.py::test_case[param_list0] - Attrib...
FAILED ascend/examples/pytest_ut/test_add.py::test_case[param_list1] - Attrib...
FAILED ascend/examples/pytest_ut/test_add.py::test_case[param_list2] - Attrib...
============================== 3 failed in 12.27s ==============================