Qwen2.5_VL_7B训练指导

docker run -d --net=host --shm-size=128g --privileged --name=qwen2_5vl \
$(for i in {0..15}; do [ -e "/dev/davinci$i" ] && echo --device=/dev/davinci$i; done) \
--device=/dev/davinci_manager \
--device=/dev/hisi_hdc \
--device=/dev/devmm_svm \
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-v /usr/local/dcmi:/usr/local/dcmi \
-e LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64:$LD_LIBRARY_PATH \
mindspeed-mm2.1.0-pytorch2.6.0-ascend_cann8.2.rc1-py310-ubuntu20.04-arm64:0905 \
/bin/bash -c "tail -f /dev/null"
/bin/bash -c "tail -f /dev/null"

docker start qwen2_5vl
docker exec -it qwen2_5vl bash

Qwen/
 └──Qwen2.5-VL-7B-Instruct
    ├──chat_template.json
    ├──config.json
    ├──configuration.json
    ├──generation_config.json
    ├──gitattributes
    ├──merges.txt
    ├──model-00001-of-00005.safetensors
    ├──model-00002-of-00005.safetensors
    ├──model-00003-of-00005.safetensors
    ├──model-00004-of-00005.safetensors
    ├──model-00005-of-00005.safetensors
    ├──model.safetensors.index.json
    ├──old-version/
    ├──preprocessor_config.json
    ├──Qwen2.5-VL-7B-Instruct.zip
    ├──README.md
    ├──tokenizer_config.json
    ├──tokenizer.json
    ├──v1/
    └──vocab.json

# 修改端口，防止端口HCCL冲突
export HCCL_IF_BASE_PORT=30000
# export HCCL_NPU_SOCKET_PORT_RANGE="6000-68050"
export HCCL_PORT_RANGE="38001-39000"
export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
# 定制修改TP、PP切分策略
TP=2
PP=2
CP=1
MBS=4

image_encoder中：
	"pipeline_num_layers": [32,0],
text_encoder中：
	"pipeline_num_layers": [8,20],

cd  /usr/local/Ascend/mm-train/MindSpeed-MM
# 生效环境变量
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh

mm-convert  Qwen2_5_VLConverter hf_to_mm \
  --cfg.mm_dir "/src/train25.10/qwen2.5-vl-7b/ckpt/mm_path/tp2_pp2" \
  --cfg.hf_config.hf_dir "base/Qwen/Qwen2.5-VL-7B-Instruct/weights/Qwen2.5-VL-7B-Instruct" \
  --cfg.parallel_config.llm_pp_layers [[8,20]] \
  --cfg.parallel_config.vit_pp_layers [[32,0]] \
  --cfg.parallel_config.tp_size 2
# 其中：
# mm_dir: 转换后保存目录
# hf_dir: huggingface权重目录
# llm_pp_layers: llm在每个卡上切分的层数，注意要和model.json中配置的pipeline_num_layers一致
# vit_pp_layers: vit在每个卡上切分的层数，注意要和model.json中配置的pipeline_num_layers一致
# tp_size: tp并行数量，注意要和微调启动脚本中的配置一致

source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
cd /usr/local/Ascend/mm-train/MindSpeed-MM/
mm-convert Qwen2_5_VLConverter mm_to_hf \
--cfg.save_hf_dir "/src/train/qwen2.5-vl-7b/mm_to_hf_save/qwen2.5hf" \
--cfg.mm_dir "/src/train25.10/qwen2.5-vl-7b/merge_save_dir/" \
--cfg.hf_config.hf_dir "/src/train25.10/hw/weights/Qwen2.5-VL-7B-Instruct/" \
--cfg.parallel_config.llm_pp_layers [8,20] \
--cfg.parallel_config.vit_pp_layers [32,0] \
--cfg.parallel_config.tp_size 2 \

# 其中：
# save_hf_dir: mm微调后转换回hf模型格式的目录
# mm_dir: 微调后保存的权重目录
# hf_dir: huggingface权重目录
# llm_pp_layers: llm在每个卡上切分的层数，注意要和微调时model.json中配置的pipeline_num_layers一致
# vit_pp_layers: vit在每个卡上切分的层数，注意要和微调时model.json中配置的pipeline_num_layers一致
# tp_size: tp并行数量，注意要和微调启动脚本中的配置一致

LOAD_PATH="/src/train25.10/qwen2.5-vl-7b/ckpt/mm_path/tp2_pp2"

[
    {
        "system": "",
        "instruction": "你是一名物流隐患识别专家，你的任务是根据输入图片识别以下四类对象的隐患：叉车、夹抱车、作业工人、纸皮箱。请根据以下图片回答问题：",
        "input": "<image>图中有几辆特种作业车？",
        "output": "2辆",
        "images": "/apps/wuliu/dataset/叉车夹抱车0516/夹抱车(数据增强9)/违规夹抱_置信度[0.0]_master1_camera-7_20250428213654.jpeg"
    },
    {
        "system": "",
        "instruction": "你是一名物流隐患识别专家，你的任务是根据输入图片识别以下四类对象的隐患：叉车、夹抱车、作业工人、纸皮箱。请根据以下图片回答问题：",
        "input": "<image>黑色车顶，蓝色车身装载有货物的特种作业车的类型是什么？",
        "output": "图中没有特种作业车",
        "images": "/apps/wuliu/dataset/叉车夹抱车0516/无叉车无夹抱车(数据增强7)/未穿反光衣_置信度[0.0]_master1_camera15_20250411171434.jpeg"
    },
    {
        "system": "",
        "instruction": "你是一名物流隐患识别专家，你的任务是根据输入图片识别以下四类对象的隐患：叉车、夹抱车、作业工人、纸皮箱。请根据以下图片回答问题：",
        "input": "<image>图像右下角区域的特种作业车的类型是什么？",
        "output": "图中没有特种作业车",
        "images": "/apps/wuliu/dataset/叉车夹抱车0516/无叉车无夹抱车(数据增强5)/破损_置信度[0.95]_master1_camera3_20250328174129.jpeg"
    },

import json
import os
from typing import Dict

PATH_PREFIX="./0622_dataset/"


def convert_path(ori_path: str) -> str:
    # 分割路径
    parts = ori_path.strip("/").split('/')
    return os.path.join(PATH_PREFIX, *parts[-3:])


def convert_alpaca_to_mindspeed_mm(alpaca_item: Dict) -> Dict:
    """单条数据转换函数（修正版）"""
    # 提取原始字段
    system_prompt = alpaca_item.get("system", "")
    instruction = alpaca_item.get("instruction", "")
    input_text = alpaca_item.get("input", "")
    output = alpaca_item.get("output", "")
    images = alpaca_item.get("images", "")

    # 构建messages列表（仅包含user和assistant）
    messages = []

    # 合并用户指令和输入
    user_content = instruction
    if input_text:
        user_content += f"\n\n{input_text}"
    messages.append({"role": "user", "content": user_content})

    # 添加助手回复
    messages.append({"role": "assistant", "content": output})

   # 图片路径处理
    images = convert_path(images)
    # 构建最终输出结构（system字段独立）
    return {
        "images":[images],
        "messages": messages,
        "system": system_prompt  # 作为顶级字段
    }


def batch_convert(input_path: str, output_path: str):
    """批量转换主函数"""
    # 读取原始数据
    with open(input_path, "r", encoding="utf-8") as f:
        alpaca_data = json.load(f)

    print("Total item num in dataset:")
    print(len(alpaca_data))
    # 转换数据
    converted_data = []
    for item in alpaca_data:
        converted_item = convert_alpaca_to_mindspeed_mm(item)
        converted_data.append(converted_item)

    # 保存结果
    os.makedirs(os.path.dirname(output_path), exist_ok=True)
    with open(output_path, "w", encoding="utf-8") as f:
        json.dump(converted_data, f, indent=4, ensure_ascii=False)

    print(f"转换完成！输出文件：{output_path}")


if __name__ == "__main__":
    # # 输入输出路径配置
    input_json = "./0622_dataset/0626_handle.json"
    output_json = "./0622_dataset/0626_gpt.json"

    # # 执行转换
    batch_convert(input_json, output_json)

python /src/train25.10/qwen2.5-vl-7b/scripts/alpaca2shareGPT.py

{
    "dataset_param": {
        "dataset_type": "huggingface",
        "preprocess_parameters": {
            "model_name_or_path": "/src/train25.10/hw/weights/Qwen2.5-VL-7B-Instruct",
             ...
        },
        "basic_parameters": {
            "template": "qwen2vl",
            "dataset_dir": "/src/train25.10/hw/wuliu/dataset",
            "dataset": "/src/train25.10/hw/wuliu/dataset/0622_dataset/0516_gpt.json",
            "cache_dir": "/src/train25.10/qwen2.5-vl-7b/data_cache_dir", 
            ...
        },
        ...
    },
    ...
}

[
    {
        "images": [
            "0622_dataset/叉车夹抱车0516/夹抱车(数据增强9)/违规夹抱_置信度[0.0]_master1_camera-7_20250428213654.jpeg"
        ],
        "messages": [
            {
                "role": "user",
                "content": "你是一名物流隐患识别专家，你的任务是根据输入图片识别以下四类对象的隐患：叉车、夹抱车、作业工人、纸皮箱。请根据以下图片回答问题：\n\n<image>图中有几辆特种作业车？"
            },
            {
                "role": "assistant",
                "content": "2辆"
            }
        ],
        "system": ""
    },
    {
        "images": [
            "0622_dataset/叉车夹抱车0516/无叉车无夹抱车(数据增强7)/未穿反光衣_置信度[0.0]_master1_camera15_20250411171434.jpeg"
        ],
        "messages": [
            {
                "role": "user",
                "content": "你是一名物流隐患识别专家，你的任务是根据输入图片识别以下四类对象的隐患：叉车、夹抱车、作业工人、纸皮箱。请根据以下图片回答问题：\n\n<image>黑色车顶，蓝色车身装载有货物的特种作业车的类型是什么？"
            },
            {
                "role": "assistant",
                "content": "图中没有特种作业车"
            }
        ],
        "system": ""
    },
    {
        "images": [
            "0622_dataset/叉车夹抱车0516/无叉车无夹抱车(数据增强5)/破损_置信度[0.95]_master1_camera3_20250328174129.jpeg"
        ],
        "messages": [
            {
                "role": "user",
                "content": "你是一名物流隐患识别专家，你的任务是根据输入图片识别以下四类对象的隐患：叉车、夹抱车、作业工人、纸皮箱。请根据以下图片回答问题：\n\n<image>图像右下角区域的特种作业车的类型是什么？"
            },
            {
                "role": "assistant",
                "content": "图中没有特种作业车"
            }
        ],
        "system": ""
    },

DP = 总调度单元数 / TP / PP / CP

Global batch size = MBS * GRAD_ACC_STEP * DP

Iteration = Epochs * 数据集条数 / Global batch size

qwen2.5-vl-7b
	├──ckpt
		└──mm_path
			└──Qwen2.5-VL-7B-Instruct            		 #切分后权重
	├──data_cache_dir                            #数据集处理后缓存目录
	├──merge_save_dir                            #合并权重保存目录
	├──logs                                      #训练日志存放目录
	├──save_dir                                  #权重训练后存放目录
	└──scripts                                   #脚本存放目录
		├──data_7b.json                            #数据目录配置文件
		├──finetune_qwen2_5_vl_7b.sh               #训练脚本
		├──mm_convert_hf_to_mm.sh                  #权重切分脚本
		├──image_first.py                          #推理测评脚本
		└──merge_lora.py                           #合并脚本

# 启动权重切分命令
cd /usr/local/Ascend/mm-train/MindSpeed-MM/ && bash /src/train25.1.0/qwen2.5-vl-7b/scripts/mm_convert_hf_to_mm.sh

# 单机启动训练命令
# 若使用公共镜像，切换工作目录至/src/train25.1.0
source /root/miniconda3/etc/profile.d/conda.sh && 
conda activate python310 && 
cd /usr/local/Ascend/mm-train/MindSpeed-MM/ && 
bash /src/train25.1.0/qwen2.5-vl-7b/scripts/finetune_qwen2_5_vl_7b.sh

# 多机启动训练命令
source /root/miniconda3/etc/profile.d/conda.sh && 
conda activate python310 && 
cd /usr/local/Ascend/mm-train/MindSpeed-MM/ && 
bash /src/train25.1.0/qwen2.5-vl-7b/scripts/multi_finetune_qwen2_5_vl_7b.sh

# 基础配置

NPUS_PER_NODE=16  #使用单节点的8卡NPU
MASTER_ADDR=$(cat /etc/volcano/worker.host | head -n 1 | awk '{print $1}') #以本节点ip地址为master_ip
echo master_addr=${MASTER_ADDR}
MASTER_PORT=6016 #本节点端口号为6015
NNODES=${VC_WORKER_NUM}  #单机，即一台节点，多机即多节点
NODE_RANK=${VC_TASK_INDEX}  #单机RANK为0，多机为(0,NNODES-1)，不同节点不可重复
WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES)) #最终使用的NPU数
echo world_size:${WORLD_SIZE}

# 该样例为TP=2，PP=2
Qwen2.5-VL-7B-Instruct
     ├──latest_checkpointed_iteration.txt
     └──release
     		├──mp_rank_00_000
     		├──mp_rank_00_001
     		├──mp_rank_01_000
				└──mp_rank_01_001

qwen2.5-vl-7b
	├──logs
		└──train_20250824_193239.log                              #训练输出日志
	├──data_cache_dir
		├──_data_share_qwen2.5-vl-7b_data_cache_dir...b1092.lock  #数据集处理后缓存文件
		├──json                                                   #数据集处理后缓存文件
	└──save_dir
		├──iter_0002464                                           #训练后模型权重文件
		├──iter_0004928                                           #训练后模型权重文件
		├──iter_0007392                                           #训练后模型权重文件
		└──latest_checkpointed_iteration.txt                      #模型检查点记录文件

vi /src/train25.10/qwen2.5-vl-7b/save_dir/latest_checkpointed_iteration.txt

vi /src/train25.10/qwen2.5-vl-7b/scripts/merge_lora.py 

...
if __name__ == '__main__':
    base_save_dir = "/src/train25.10/qwen2.5-vl-7b/ckpt/mm_path/Qwen2.5-VL-7B-Instruct"
    lora_save_dir = "/src/train25.10/qwen2.5-vl-7b/save_dir/"
    merge_save_dir = "/src/train25.10/qwen2.5-vl-7b/merge_save_dir/"
    lora_target_modules = ['linear_qkv', 'linear_proj', 'linear_fc1', 'linear_fc2']

    lora_alpha = 12
    lora_r = 6          # 保持和训练一致
    scaling = lora_alpha / lora_r

    pp_size = 1
    tp_size = 2
...    

export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/lib64:$LD_LIBRARY_PATH && 
export PATH=/usr/local/Ascend/ascend-toolkit/latest/bin:$PATH && 
source /usr/local/Ascend/ascend-toolkit/set_env.sh && 
python /src/train25.10/qwen2.5-vl-7b/scripts/merge_lora.py 

cd  /usr/local/Ascend/mm-train/MindSpeed-MM
# 生效环境变量
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh

mm-convert Qwen2_5_VLConverter mm_to_hf \
    --cfg.save_hf_dir "/src/train25.10/qwen2.5-vl-7b/mm_to_hf_save/" \
    --cfg.mm_dir "/src/train25.10/qwen2.5-vl-7b/merge_save_dir" \
    --cfg.hf_config.hf_dir "/src/train25.10/hw/weights/Qwen2.5-VL-7B-Instruct" \
    --cfg.parallel_config.llm_pp_layers [28] \
    --cfg.parallel_config.vit_pp_layers [32] \
    --cfg.parallel_config.tp_size 2
    
# 其中：
# save_hf_dir: mm微调后转换回hf模型格式的目录
# mm_dir: 微调后保存的权重目录
# hf_dir: huggingface权重目录
# llm_pp_layers: llm在每个卡上切分的层数，注意要和微调时model.json中配置的pipeline_num_layers一致
# vit_pp_layers: vit在每个卡上切分的层数，注意要和微调时model.json中配置的pipeline_num_layers一致
# tp_size: tp并行数量，注意要和微调启动脚本中的配置一致

cd /hw_test/hw/wuliu/dataset/

# 修改验证脚本加载权重路径，指定验证集：vi image_first.py

python image_first.py    # image_first.py是自定义的推理脚本，需要权重转回Huggingface格式

bash eval.sh --step 500

#!/bin/bash

# ==============================================

# LoRA Weight Version Automation Script

# Usage: ./eval.sh <version_number> 

# Example: ./eval.sh 1500

# ==============================================

# Validate input argument

if [ $# -eq 0 ]; then
    echo "ERROR: Version number argument missing"
    echo "Usage: $0 <version_number> (e.g. 1500)"
    exit 1
fi
DEFAULT_VERSION="2000"
VERSION="${1:-$DEFAULT_VERSION}"
CHECKPOINT_FILE="/src/train25.10/qwen2.5-vl-7b/save_dir/2node/latest_checkpointed_iteration.txt"

echo "=== STARTING PIPELINE FOR VERSION: $VERSION ==="

source /usr/local/Ascend/ascend-toolkit/set_env.sh

# -------------------------------------------------------------------

# STEP 1: Overwrite version number in checkpoint file

# -------------------------------------------------------------------

echo "[1/4] Writing version $VERSION to $CHECKPOINT_FILE..."
echo "$VERSION" > "$CHECKPOINT_FILE" || {
    echo "ERROR: Failed to write to $CHECKPOINT_FILE"
    exit 1
}

# -------------------------------------------------------------------

# STEP 2: Run LoRA weight merging script

# -------------------------------------------------------------------

echo "[2/4] Executing merge_lora.py..."
cd  /usr/local/Ascend/mm-train/MindSpeed-MM
python examples/qwen2vl/merge_lora.py || {
    echo "ERROR: merge_lora.py execution failed"
    exit 1
}

# -------------------------------------------------------------------

# STEP 3: Convert model format using mm-convert

# -------------------------------------------------------------------

echo "[3/4] Running mm-convert for version $VERSION..."
mm-convert Qwen2_5_VLConverter mm_to_hf \
    --cfg.save_hf_dir "/src/train25.10/qwen2.5-vl-7b/mm_to_hf_save/$VERSION" \
    --cfg.mm_dir "/src/train25.10/qwen2.5-vl-7b/merge_save_dir" \
    --cfg.hf_config.hf_dir "/src/train25.10/hw/weights/Qwen2.5-VL-7B-Instruct" \
    --cfg.parallel_config.llm_pp_layers [28] \
    --cfg.parallel_config.vit_pp_layers [32] \
    --cfg.parallel_config.tp_size 2 || {
    echo "ERROR: mm-convert operation failed"
    exit 1
}

# -------------------------------------------------------------------

# STEP 4: Process dataset with version-specific parameter

# -------------------------------------------------------------------

echo "[4/4] Running image_first.py with step=$VERSION..."
cd /src/train25.10/qwen2.5-vl-7b/scripts || {
    echo "ERROR: Failed to access dataset directory"
    exit 1
}
python image_first.py --step "$VERSION" || {
    echo "ERROR: image_first.py execution failed"
    exit 1
}

# -------------------------------------------------------------------

# SUCCESS OUTPUT

# -------------------------------------------------------------------

echo "=== PIPELINE COMPLETED SUCCESSFULLY FOR VERSION $VERSION ==="
exit 0