使用Mindspeed-MM训练框架微调Qwen2.5VL-7B模型指导

[
    {
        "id": "20251017_10018_00001",
        "image": "20251017_10018_00001.jpg",
        "conversations": [
            {
                "from": "human",
                "value": "<image>\n请提取这张存单图片中的存单号码和账号号码."
            },
            {
                "from": "gpt",
                "value": "存单号码: 2201361629 账号号码: 51050006574529"
            }
        ]
    },
    {
        "id": "20251017_10018_00002",
        "image": "20251017_10018_00002.jpg",
        "conversations": [
            {
                "from": "human",
                "value": "<image>\n请提取这张存单图片中的存单号码和账号号码."
            },
            {
                "from": "gpt",
                "value": "存单号码: 2201361629 账号号码: 51050006574529"
            }
        ]
    }
]

# llava格式QA标注文件路径
llava_json_path = "./data/task1/train_llava_format.json"
# mllm格式QA标注文件路径
mllm_format_json_path = "./data/task1/train_mllm_format.json"
# 训练图片路径
img_path = os.path.join("./data/train", item["image"])

[{"images": ["./img\\20251017_10018_00001.jpg"], "messages": [{"role": "user", "content": "<image>\n请提取这张存单图片中的存单号码和账号号码."}, {"role": "assistant", "content": "存单号码: 2201361629 账号号码: 51050006574529"}]}, {"images": ["./img\\20251017_10018_00002.jpg"], "messages": [{"role": "user", "content": "<image>\n请提取这张存单图片中的存单号码和账号号码."}, {"role": "assistant", "content": "存单号码: 2201361629 账号号码: 51050006574529"}]}]

"basic_parameters": {
            "template": "qwen2vl",
            "dataset_dir": "./data",
            "dataset": "./data/trian_mllm_format.json",
            "cache_dir": "./data/cache_dir",
            "train_on_prompt": false,
            "mask_history": false,
            "preprocessing_batch_size": 1000,
            "preprocessing_num_workers": 16,
            "max_samples": null,
            "tool_format": null
        }

# 默认使用8卡
NPUS_PER_NODE=8
MASTER_ADDR=localhost
MASTER_PORT=6000
NNODES=1
NODE_RANK=0
WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))

# 默认加载data_7b.json
MM_DATA="./examples/qwen2.5vl/data_7b.json"
MM_MODEL="./examples/qwen2.5vl/model_7b.json"
MM_TOOL="./mindspeed_mm/tools/tools.json"
# 按需修改你的权重路径
LOAD_PATH="ckpt/mm_path/Qwen2.5-VL-7B-Instruct"
# 按需修改你的ckpt保存路径
SAVE_PATH="output"

# train-iters 按需修改迭代次数，原始代码默认10000次
GPT_ARGS="
    --use-mcore-models \
    --tensor-model-parallel-size ${TP} \
    --pipeline-model-parallel-size ${PP} \
    --context-parallel-size ${CP} \
    --context-parallel-algo ulysses_cp_algo \
    --micro-batch-size ${MBS} \
    --global-batch-size ${GBS} \
    --tokenizer-type NullTokenizer \
    --vocab-size 152064 \
    --seq-length 1024 \
    --make-vocab-size-divisible-by 1 \
    --normalization RMSNorm \
    --use-fused-rmsnorm \
    --swiglu \
    --use-fused-swiglu \
    --no-masked-softmax-fusion \
    --lr 1.0e-5 \
    --lr-decay-style cosine \
    --weight-decay 0 \
    --train-iters 10000 \
    --lr-warmup-fraction 0.1 \
    --clip-grad 0.0 \
    --adam-beta1 0.9 \
    --adam-beta2 0.999 \
    --seed 42 \
    --bf16 \
    --load $LOAD_PATH \
    --use-flash-attn \
    --variable-seq-lengths \
    --use-distributed-optimizer \
    --no-load-optim \
    --no-load-rng \
    --no-save-optim \
    --no-save-rng \
    --num-workers 8 \
    --distributed-timeout-minutes 20 \
"
# 按需修改 save-interval
OUTPUT_ARGS="
    --log-interval 1 \
    --save-interval 10000 \
    --eval-interval 10000 \
    --eval-iters 5000 \
    --save $SAVE_PATH \
    --ckpt-format torch \
    --log-tps \
"

docker load -i image.tar

docker run -itd  \
--privileged=true \
--name mindspeed_mm \
--network=host \
--pid=host \
--detach=true \
--shm-size=64g \
--device=/dev/davinci0 \
--device=/dev/davinci1 \
--device=/dev/davinci2 \
--device=/dev/davinci3 \
--device=/dev/davinci4 \
--device=/dev/davinci5 \
--device=/dev/davinci6 \
--device=/dev/davinci7 \
--device /dev/davinci_manager \
--device /dev/devmm_svm \
--device /dev/hisi_hdc \
-v /usr/local/sbin:/usr/local/sbin \
-v /usr/local/dcmi:/usr/local/dcmi \
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-v /etc/ascend_install.info:/etc/ascend_install.info \
-v /root/.cache:/root/.cache \
-v /tmp:/tmp \
-v /data-VL/:/data-VL/ \
vlm_image:tag /bin/bash

docker exec -it mindspeed_mm bash
# 容器内，工作目录位于MindSpeed-MM/
cd /data-VL/MindSpeed-MM/
# 启动微调
bash examples/qwen2.5vl/finetune_qwen2_5_vl_7b.sh

[2025-10-29 02:21:07] iteration 28/1500 | consumed samples : 5376 | elapsed time per iteration (ms): 7166.4 | learning rate: 1.866667E-06 | global batch size: 192 | loss: 2.061336E-02 | loss scale: 1.0 | grad norm: 1.032 | num zeros: 0.0 | number of skipped iterations:0 | number of nan iterations: 0
[2025-10-29 02:21:14] iteration 29/1500 | consumed samples : 5568 | elapsed time per iteration (ms): 7179.2 | learning rate: 1.933333E-06 | global batch size: 192 | loss: 2.999549E-02 | loss scale: 1.0 | grad norm: 0.712 | num zeros: 0.0 | number of skipped iterations:0 | number of nan iterations: 0

[2025-10-29 05:20:20] iteration 1500/1500 | consumed samples : 288000 | elapsed time per iteration (ms): 7342.6 | learning rate: 0.000000E+00 | global batch size: 192 | loss: 3.903987E-07 | loss scale: 1.0 | grad norm: 0.000 | num zeros: 0.0 | number of skipped iterations:0 | number of nan iterations: 0

saving checkpoint at iteration 1500 to ./output in torch format
successfully saved checkpoint from iteration 1500 to ./output [ t 1/1, p 1/2 ]
[after training is done] datetime: 2025-10-29 05:20:32
Elapsed Time Per iteration: 7249.3
Average Samples per Second: 26.485

mm-convert Qwen2_5_VLConverter mm_to_hf \
-- cfg.save_hf_dir "output-mm2hf" \
-- cfg.mm_dir "output" \
-- cfg.hf_config.hf_dir "ckpt/hf_path/Qwen2.5-VL-7B-Instruct" \
-- cfg.parallel_config.1lm_pp_layers [12,16] \
-- cfg.parallel_config.vit_pp_layers [32,0]
-- cfg.parallel_config.tp_size 1

cd MindSpeed-MM && cp $SCRIPT_PATH/* ./ && cp $SCRIPT_PATH/finetune_qwen2_5_vl_7b.sh examples/qwen2.5vl/ && bash run.sh

cp -r $DATA_PATH/* data/ && mv data/data_7b.ison examples/qwen2.5vl/ && python llava_2_mllm_format.py --llava-path data/train_llava.json --mllm-path data/train_mllm.json --img-dir data/train_img && bash examples/qwen2.5vl/finetune_qwen2_5_vl_7b.sh && bash mm-convert-mm2hf.sh

import argparse
import json
import os
import stat

def convert_llava_to_mllm_format(llava_json_path, mllm_format_json_path, img_dir_path):
    with open(llava_json_path, "r") as f:
        info_json = json.loads(f.read())

    mllm_format_llava_instruct_data = []
    for item in info_json:
        # 处理图像路径
        if not item.get('image'):
            new_item = {
                "images": [],
                "messages": []
            }
        else:
            img_path = os.path.join(img_dir_path, item["image"])
            print(f"img_path: {img_path}")
            if not os.path.exists(img_path):
                continue
            new_item = {
                "images": [img_path],
                "messages": []
            }

        # 处理对话内容
        for turn in item["conversations"]:
            if turn["from"] == "human":
                new_item["messages"].append({"role": "user", "content": turn["value"]})
            elif turn["from"] == "gpt":
                new_item["messages"].append({"role": "assistant", "content": turn["value"]})
            else:
                raise ValueError(f"unknown role: {turn['from']}")
        mllm_format_llava_instruct_data.append(new_item)

    # 写入输出文件
    output_json = json.dumps(mllm_format_llava_instruct_data, ensure_ascii=False)
    if os.path.exists(mllm_format_json_path):
        print(f"{mllm_format_json_path} already exists, please rename it or remove it")
        return
    with os.fdopen(os.open(mllm_format_json_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, stat.S_IWUSR | stat.S_IRUSR), "w") as f:
        f.write(output_json)
    print(f"finish converting dataset into {mllm_format_json_path}")

# 使用示例（可根据实际需求修改路径参数）
if __name__ == "__main__":
    # 解析命令行参数
    parser = argparse.ArgumentParser(description="Convert LLaVA format to MLLM format")
    parser.add_argument("--llava-path", required=True, help="Path to LLaVA JSON file")
    parser.add_argument("--mllm-path", required=True, help="Path to save MLLM format JSON")
    parser.add_argument("--img-dir", required=True, help="Directory containing images")

    args = parser.parse_args()

    # 调用函数
    convert_llava_to_mllm_format(
        llava_json_path=args.llava_path,
        mllm_format_json_path=args.mllm_path,
        img_dir_path=args.img_dir
    )