引言

# 增加软件包可执行权限，{version}表示软件版本号，{arch}表示CPU架构，{soc}表示昇腾AI处理器的版本。
chmod +x ./Ascend-cann-toolkit_{version}_linux-{arch}.run
chmod +x ./Ascend-cann-kernels-{soc}_{version}_linux.run
chmod +x ./Ascend-cann-nnal_{version}_linux-{arch}.run
# 校验软件包安装文件的一致性和完整性
./Ascend-cann-toolkit_{version}_linux-{arch}.run --check
./Ascend-cann-kernels-{soc}_{version}_linux.run --check
./Ascend-cann-nnal{version}_linux-{arch}.run --check
# 安装
./Ascend-cann-toolkit_{version}_linux-{arch}.run --install
./Ascend-cann-kernels-{soc}_{version}_linux.run --install
./Ascend-cann-nnal{version}_linux-{arch}.run --torch_atb --install

# 设置环境变量
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh

docker load -i v0.11.0rc2.tar

docker images | grep v0.11.0rc2

export model_dir="/opt/data/modelscope/models"     # 宿主机保存模型权重的路径
export container_model_dir="/root/models"          # 映射到容器内保存模型权重的路径
export container_name="Qwen3-VL"                   # 指定容器名称
export IMAGE=m.daocloud.io/quay.io/ascend/vllm-ascend:v0.11.0rc2   # 容器镜像的名称
docker run -itd --privileged \
--name ${container_name} \
--net=host \
--device /dev/davinci_manager \
--device /dev/devmm_svm \
--device /dev/hisi_hdc \
--shm-size=512g \
-v /usr/local/dcmi:/usr/local/dcmi \
-v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool \
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-v /etc/ascend_install.info:/etc/ascend_install.info \
-v ${model_dir}:${container_model_dir} \
$IMAGE /bin/bash

docker exec -it -u root Qwen3-VL bash

# 通过如下ASCEND_RT_VISIBLE_DEVICES环境变量指定使用的NPU卡设备
export ASCEND_RT_VISIBLE_DEVICES=3,7

# 如下/root/models/Qwen/Qwen3-VL-30B-A3B-Thinking 为容器内权重文件的路径，qwen3-vl-30b为指定的模型实例名称
nohup vllm serve /root/models/Qwen/Qwen3-VL-30B-A3B-Thinking \
  --host 0.0.0.0 \
  --port 33030 \
  --tensor-parallel-size 2 \
  --enable-expert-parallel \
  --max-num-seqs 16 \
  --max-num-batched-tokens 4096 \
  --trust-remote-code \
  --no-enable-prefix-caching \
  --gpu-memory-utilization 0.8 \
  --max-model-len 32768 \
  --served-model-name qwen3-vl-30b > ./qwen3-vl-30b.log 2>&1 &

curl http://{IP地址:端口号}/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
    "model": "qwen3-vl-30b",
    "messages": [
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": [
        {"type": "image_url", "image_url": {"url": "https://modelscope.oss-cn-beijing.aliyuncs.com/resource/qwen.png"}},
        {"type": "text", "text": "What is the text in the illustrate?"}
        ]}
    ]
}'

# 文件名 test.py
import requests
import json
import base64

def image_to_base64(file_path):
    with open(file_path, "rb") as f:
        image_byte = f.read()
    return base64.b64encode(image_byte).decode('utf-8')

base64_str = image_to_base64("{本地图片路径}")    # 指定一张本地的图片
url = "http://{IP地址:端口号}/v1/chat/completions"
headers = {
    "Content-Type": "application/json"
}
data = {
    "model": "qwen3-vl-30b",  # 确保mode名称与启动推理服务命令中的served-model-name参数一致
    "messages": [
        {
            "role": "user",
            "content": [
                {"type": "image_url", "image_url": {"url": f"data:image/jpg;base64,{base64_str}"}},
                {"type": "text", "text": "Explain the details in the image."}
            ]
        }
    ],
    "max_tokens": 100,
    "temperature": 0.7
}

response = requests.post(url, headers=headers, data=json.dumps(data))

if response.status_code == 200:
    result = response.json()
    print(result['choices'][0]['message']['content'])
else:
    print(f"请求失败，状态码: {response.status_code}, 响应内容: {response.text}")