docker run \
  --name vllm_ascend_v0.20 \
  --network host \
  -d \
  --shm-size=1g \
  --device=/dev/davinci_manager \
  --device=/dev/hisi_hdc \
  --device=/dev/devmm_svm \
  --device=/dev/davinci0 \
  --device=/dev/davinci1 \
  --device=/dev/davinci2 \
  --device=/dev/davinci3 \
  --device=/dev/davinci4 \
  --device=/dev/davinci5 \
  --device=/dev/davinci6 \
  --device=/dev/davinci7 \
  -v /usr/local/Ascend/driver:/usr/local/Ascend/driver:ro \
  -v /usr/local/sbin:/usr/local/sbin:ro \
  -v /data/weights:/data/weights \
  quay.io/ascend/vllm-ascend:v0.20.2rc1 \
  bash /data/weights/DeepSeek-V4-Flash-w8a8-mtp/start_vllm_ascend_10.1.4.3_vllm_ascend.sh

export PYTHONPATH=/vllm-workspace/vllm:$PYTHONPATH

export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD
export OMP_PROC_BIND=false
export OMP_NUM_THREADS=10
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export ACL_OP_INIT_MODE=1
export VLLM_ASCEND_ENABLE_FLASHCOMM1=1

export USE_MULTI_GROUPS_KV_CACHE=1

export TASK_QUEUE_ENABLE=1
export HCCL_OP_EXPANSION_MODE="AIV"
export HCCL_BUFFSIZE=512

export USE_MULTI_BLOCK_POOL=1


sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl kernel.sched_migration_cost_ns=50000


# Redirect output to log file for debugging
LOG_FILE="/data/weights/DeepSeek-V4-Flash-w8a8-mtp/vllm_ascend_10.1.4.3_vllm_ascend.log"
> "$LOG_FILE" 2>&1
exec > >(tee -a "$LOG_FILE") 2>&1

echo "Starting vllm at $(date)"

vllm serve /data/weights/DeepSeek-V4-Flash-w8a8-mtp \
        --port 8080 \
        --safetensors-load-strategy 'prefetch' \
        --max-model-len 10240 \
        --max-num-batched-tokens 10240 \
        --gpu-memory-utilization 0.9 \
        --max-num-seqs 32 \
        --data-parallel-size 1 \
        --tensor-parallel-size 8 \
        --enable-expert-parallel \
        --quantization ascend \
        --block-size 128 \
        --enable-chunked-prefill \
        --no-enable-prefix-caching \
        --tokenizer-mode deepseek_v4 \
        --tool-call-parser deepseek_v4 \
        --enable-auto-tool-choice \
        --reasoning-parser deepseek_v4 \
        --async-scheduling \
        --chat-template /data/weights/DeepSeek-V4-Flash-w8a8-mtp/chat_template.jinja \
        --compilation-config '{
        "cudagraph_mode": "FULL_DECODE_ONLY"
        }' \
        --additional-config '{
                "ascend_compilation_config":{
                "enable_npugraph_ex":true,
                "enable_static_kernel":false
            },
            "enable_cpu_binding": "true",
            "multistream_overlap_shared_expert":false,
            "multistream_dsa_preprocess":false
        }' \
        --speculative-config '{
            "num_speculative_tokens": 1,
            "method": "mtp"}
        ' \
        --served-model-name DeepSeek-V4-Flash-w8a8-mtp

(Worker_TP0_EP0 pid=314) ERROR 06-06 07:56:49 [multiproc_executor.py:962]   File "/vllm-workspace/vllm-ascend/vllm_ascend/worker/model_runner_v1.py", line 4447, in _torch_cuda_wrapper^M
(Worker_TP0_EP0 pid=314) ERROR 06-06 07:56:49 [multiproc_executor.py:962]     raise RuntimeError(f"NPUModelRunner init failed, error is {e}")^M
(Worker_TP0_EP0 pid=314) ERROR 06-06 07:56:49 [multiproc_executor.py:962] RuntimeError: NPUModelRunner init failed, error is NPUModelRunner failed, error is ^M
(Worker_TP0_EP0 pid=314) ERROR 06-06 07:56:49 [multiproc_executor.py:962] Traceback (most recent call last):^M