docker pull quay.io/ascend/vllm-ascend:v0.11.0rc2

pip install modelscope
modelscope download --model Qwen/Qwen2.5-VL-3B-Instruct --local_dir ./Qwen2.5-VL-3B-Instruct

# Qwen2.5-VL requires the latest transformers (transformers >= 4.49.0)，使用国内镜像源

pip uninstall -y transformers accelerate

pip install transformers==4.49.0 accelerate==0.27.0 -i https://pypi.tuna.tsinghua.edu.cn/simple 

# Qwen vl utils

pip install qwen-vl-utils==0.0.8

vim one_pic_inference.py

from qwen_vl_utils import process_vision_info

model = Qwen2_5_VLForConditionalGeneration.from_pretrained(

"xxx/Qwen2.5-VL-3B-Instruct", torch_dtype="auto", device_map="auto"

)

processor = AutoProcessor.from_pretrained("xxx/Qwen2.5-VL-3B-Instruct")

# The default range for the number of visual tokens per image in the model is 4-16384.

# You can set min_pixels and max_pixels according to your needs, such as a token range of 256-1280, to balance performance and cost.

# min_pixels = 256*28*28

# max_pixels = 1280*28*28

# processor = AutoProcessor.from_pretrained("Qwen/Qwen2.5-VL-3B-Instruct", min_pixels=min_pixels, max_pixels=max_pixels)

messages = [

 {

 "role": "user",

 "content": [

 {

 "type": "image",

 "image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg",

 },

 {"type": "text", "text": "Describe this image."},

 ],

 }

]

# Preparation for inference

text = processor.apply_chat_template(

 messages, tokenize=False, add_generation_prompt=True

)

image_inputs, video_inputs = process_vision_info(messages)

inputs = processor(

 text=[text],

 images=image_inputs,

 videos=video_inputs,

 padding=True,

 return_tensors="pt",

)

inputs = inputs.to("npu")



# Inference: Generation of the output

generated_ids = model.generate(**inputs, max_new_tokens=128)

generated_ids_trimmed = [

 out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)

]

output_text = processor.batch_decode(

 generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False

)

print(output_text)

vim multi_pics_inference.py

from modelscope import Qwen2_5_VLForConditionalGeneration, AutoTokenizer, AutoProcessor
from qwen_vl_utils import process_vision_info
import torch
from PIL import Image

# ==============================
# 1. 模型加载（自动分配到两个 NPU 卡）
# ==============================
# 替换原来的 model 加载代码：
model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
"xxx/Qwen2.5-VL-3B-Instruct",
torch_dtype="auto",
# device_map="auto" ←【删除或注释这一行】
)

# 显式指定使用单个 NPU（例如 NPU 0）
device = "npu:0"
model = model.to(device)

# ==============================
# 2. 加载 Processor（支持多图）
# ==============================
processor = AutoProcessor.from_pretrained(
    "xxx/Qwen2.5-VL-3B-Instruct"
)

# ==============================
# 3. 准备 3 张本地图片路径（替换为你的图片路径）
# ==============================
image_paths = [
    "xxx/car1.jpg",  # 替换为真实路径
    "xxx/car2.jpeg",
    "xxx/car3.jpg"
]

# 加载图片为 PIL.Image 对象
images = []
for path in image_paths:
    img = Image.open(path).convert("RGB")
    # 降低分辨率（关键！）
    img = img.resize((768, 768), Image.Resampling.LANCZOS)
    images.append(img)


# ==============================
# 4. 构建 messages（包含 3 张图像 + 文本提问）
# ==============================
messages = [
    {
        "role": "user",
        "content": [
            {
                "type": "image",
                "image": img,  # 直接传入 PIL.Image 对象
            }
            for img in images
        ] + [
            {"type": "text", "text": "Describe all three images in detail."}
        ],
    }
]

# ==============================
# 5. 处理对话模板 + 提取图像输入
# ==============================
text = processor.apply_chat_template(
    messages, tokenize=False, add_generation_prompt=True
)

image_inputs, video_inputs = process_vision_info(messages)

# ==============================
# 6. 构造输入并移动到 NPU
# ==============================
inputs = processor(
    text=[text],
    images=image_inputs,   # 支持多个图像（列表）
    videos=video_inputs,
    padding=True,
    return_tensors="pt",
)

# 移动到 NPU（自动跨卡）
inputs = inputs.to("npu")

# ==============================
# 7. 推理生成
# ==============================
generated_ids = model.generate(**inputs, max_new_tokens=512)

# 去掉输入部分
generated_ids_trimmed = [
    out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
]

# 解码输出
output_text = processor.batch_decode(
    generated_ids_trimmed,
    skip_special_tokens=True,
    clean_up_tokenization_spaces=False
)

print("✅ 推理结果：")
print(output_text[0])

python multi_pics_inference.py