Qwen3-VL-Embedding-2B 昇腾NPU适配

Qwen3-VL-embedding-2B/
├── README.md                              # 本文件
├── Qwen3-VL-Embedding-2B-NPU-适配指导书.md  # 详细适配指导书
├── verify_report.md                        # 验证报告
├── training_report.md                      # 微调训练报告
├── accuracy_comparison_report.md           # 精度对比报告
├── scripts/                                # 脚本目录
│   ├── offline_inference.py                # 离线推理脚本
│   ├── image_accuracy_eval.py              # 多模态精度评测脚本
│   ├── start_server.sh                     # 在线服务启动脚本
│   ├── verify_server.sh                    # 服务验证脚本
│   ├── convert_flickr8k.py                 # 数据集转换脚本
│   ├── train_contrastive_simple.py         # 对比学习微调脚本
│   └── evaluate_finetuned_model.py         # 微调后模型评测脚本
│   └── convert_weights.sh                  # MindSpeed-MM权重转换
├── mindspeed_mm/                           # MindSpeed-MM微调适配代码
│   ├── models/
│   │   └── embedding_model.py              # Embedding模型类
│   ├── tasks/
│   │   └── contrastive/
│   │       └── contrastive_trainer.py      # 对比学习训练器
│   └── data/
│       ├── datasets/
│       │   └── contrastive_dataset.py      # 对比学习数据集
│       └── dataloader/
│           └── data_collator.py            # 数据Collator
├── examples/
│   └── qwen3vl_embedding/                  # MindSpeed-MM微调配置
│       ├── qwen3vl_embedding_sft.yaml      # 训练配置
│       ├── model_config.json               # 模型配置
│       ├── data_config.json                # 数据配置
│       ├── finetune_qwen3vl_embedding.sh   # 启动脚本
└── data/
    └── flickr8k_contrastive_sample.json    # 示例数据

import torch
import torch_npu
from vllm import LLM

os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "3"

# 任务描述
task = 'Given a web search query, retrieve relevant passages that answer the query'

# 构造查询和文档
queries = [
    f'Instruct: {task}\nQuery: What is the capital of China?',
    f'Instruct: {task}\nQuery: Explain gravity'
]
documents = [
    "The capital of China is Beijing.",
    "Gravity is a force that attracts two bodies towards each other."
]
input_texts = queries + documents

# 加载模型
model = LLM(
    model="/workspace/Qwen3-VL-Embedding-2B",
    runner="pooling",
    distributed_executor_backend="mp",
    trust_remote_code=True
)

# 计算嵌入
outputs = model.embed(input_texts)
embeddings = torch.tensor([o.outputs.embedding for o in outputs])

# 计算相似度
scores = (embeddings[:2] @ embeddings[2:].T)
print(f"Embedding scores: {scores.tolist()}")
print(f"Embedding shape: {embeddings.shape}")

Embedding scores: [[0.91, 0.29], [0.30, 0.78]]
Embedding shape: torch.Size([4, 2048])

python scripts/offline_inference.py

export ASCEND_RT_VISIBLE_DEVICES="3"

vllm serve /workspace/Qwen3-VL-Embedding-2B \
    --runner pooling \
    --port 9008 \
    --served-model-name Qwen3-VL-Embedding-2B \
    --trust-remote-code

# 检查模型列表
curl http://127.0.0.1:9008/v1/models

# 发送推理请求
curl http://127.0.0.1:9008/v1/embeddings \
  -H "Content-Type: application/json" \
  -d '{
    "input": ["The capital of China is Beijing.", "Gravity is a force."],
    "model": "Qwen3-VL-Embedding-2B"
  }'

bash scripts/start_server.sh
bash scripts/verify_server.sh

python scripts/image_accuracy_eval.py --num_samples 100

python scripts/convert_flickr8k.py

python scripts/train_contrastive_simple.py \
    --model /workspace/Qwen3-VL-Embedding-2B \
    --data /workspace/MindSpeed-MM/data/flickr8k_contrastive.jsonl \
    --output /workspace/outputs/qwen3vl_embedding_finetuned \
    --epochs 3 \
    --batch_size 4 \
    --max_samples 500 \
    --lr 2e-5 \
    --temperature 0.07

git clone https://gitcode.com/Ascend/MindSpeed-MM.git
pip install -e .

cp -r mindspeed_mm/* MindSpeed-MM/mindspeed_mm/
cp -r examples/* MindSpeed-MM/examples/

bash examples/qwen3vl_embedding/finetune_qwen3vl_embedding.sh

python scripts/evaluate_finetuned_model.py \
    --model /workspace/outputs/qwen3vl_embedding_finetuned/final \
    --num_samples 50

from transformers import Qwen3VLForConditionalGeneration, AutoProcessor
from qwen_vl_utils import process_vision_info
import torch.nn.functional as F

model = Qwen3VLForConditionalGeneration.from_pretrained(
    "/workspace/Qwen3-VL-Embedding-2B",
    torch_dtype=torch.bfloat16,
    device_map="auto"
)
processor = AutoProcessor.from_pretrained("/workspace/Qwen3-VL-Embedding-2B")

# 图像嵌入
messages = [
    {"role": "user", "content": [{"type": "image", "image": "path/to/image.jpg"}]}
]
text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
image_inputs, _ = process_vision_info(messages)
inputs = processor(text=[text], images=image_inputs, return_tensors="pt").to(model.device)

with torch.no_grad():
    outputs = model(**inputs, output_hidden_states=True)
    hidden = outputs.hidden_states[-1]
    # 从最后一个token提取嵌入
    emb = hidden[:, -1, :]
    emb = F.normalize(emb, p=2, dim=-1)