Qwen2.5-3B-Instruct-2k-MindspeedLLM训练指导

# 业务数据集预处理
source /usr/local/Ascend/ascend-toolkit/set_env.sh
cd /usr/local/Ascend/llm-train/MindSpeed-LLM
datapath=***
mkdir ${datapath}/train_data_10k

python ./preprocess_data.py \
    --input ${datapath}/train_data_10k.jsonl \
    --tokenizer-name-or-path /data/***/Qwen2.5-3B-Instruct \
    --output-prefix ${datapath}/train_data_10k/train_data_10k \
    --workers 4 \
    --log-interval 1000 \
    --tokenizer-type PretrainedFromHF \
    --handler-name SharegptStyleInstructionHandler \
    --prompt-type qwen \
    --map-keys '{"messages":"messages", "tags":{"role_tag": "role","content_tag": "content","user_tag": "user","assistant_tag": "assistant","system_tag": "system"}}'

# 切换MindspeedLLM路径
cd /src/train25.1.0/MindSpeed-LLM
source /usr/local/Ascend/ascend-toolkit/set_env.sh

echo "============== start convert_ckpt ================"
# 设置需要的权重转换参数
python convert_ckpt.py \
       --use-mcore-models \
       --model-type GPT \
       --load-model-type hf \
       --save-model-type mg \
       --target-tensor-parallel-size 2 \
       --target-pipeline-parallel-size 1 \
       --add-qkv-bias \
       --load-dir /data/***/Qwen2.5-3B-Instruct/ \
       --save-dir /data/private/models_mg/Qwen2.5-3B-Instruct_tp2_pp1/ \
       --tokenizer-model  /data/***/Qwen2.5-3B-Instruct/tokenizer.json \
       --model-type-hf llama2 \
       --params-dtype bf16

#!/bin/bash
# 脚本名：/data/private/scripts/llm/qwen25-3b-instruct/tune_qwen25_3b_4k_full_ptd.sh
cd /src/train25.1.0/MindSpeed-LLM
source /usr/local/Ascend/ascend-toolkit/set_env.sh

export HCCL_CONNECT_TIMEOUT=1200
export CUDA_DEVICE_MAX_CONNECTIONS=1
export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15

NPUS_PER_NODE=16
MASTER_ADDR=localhost
MASTER_PORT=2901
NNODES=1
NODE_RANK=0
WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))

mkdir -p /data/private/outputs/llm/qwen2.5-3b-instruct/logs
echo "============== start train ================"

# please fill these path configurations
CKPT_LOAD_DIR="/data/private/models_mg/Qwen2.5-3B-Instruct_tp2_pp1/"
CKPT_SAVE_DIR="/data/private/models_mg/ckpt/Qwen2.5-3B-Instruct_tp2_pp1/"
DATA_PATH=""
TOKENIZER_PATH="/data/***/Qwen2.5-3B-Instruct"
tensorboard_dir="/data/private/outputs/llm/qwen2.5-3b-instruct/logs/tensorboard_dir/full"
train_log_path="/data/private/outputs/llm/qwen2.5-3b-instruct/logs/tune_mcore_qwen25_3b_full_tp2_pp1_train.log"

TP=2
PP=1
MBS=8
GBS=128
SEQ_LEN=2048
TRAIN_ITERS=400

DISTRIBUTED_ARGS="
    --nproc_per_node $NPUS_PER_NODE \
    --nnodes $NNODES \
    --node_rank $NODE_RANK \
    --master_addr $MASTER_ADDR \
    --master_port $MASTER_PORT
"
    
GPT_ARGS="
    --finetune \
    --stage sft \
    --is-instruction-dataset \
    --tokenizer-not-use-fast \
    --prompt-type qwen \
    --variable-seq-lengths \
    --use-mcore-models \
    --tensor-model-parallel-size ${TP} \
    --pipeline-model-parallel-size ${PP} \
    --num-layers 36 \
    --hidden-size 2048 \
    --ffn-hidden-size 11008 \
    --num-attention-heads 16 \
    --group-query-attention \
    --num-query-groups 2 \
    --tokenizer-type PretrainedFromHF \
    --tokenizer-name-or-path ${TOKENIZER_PATH} \
    --seq-length ${SEQ_LEN} \
    --max-position-embeddings ${SEQ_LEN} \
    --micro-batch-size ${MBS} \
    --global-batch-size ${GBS} \
    --make-vocab-size-divisible-by 1 \
    --padded-vocab-size 151936 \
    --rotary-base 1000000 \
    --train-iters ${TRAIN_ITERS} \
    --lr 1.0e-5 \
    --weight-decay 0.0 \
    --lr-decay-style cosine \
    --clip-grad 1.0 \
    --adam-beta1 0.9 \
    --adam-beta2 0.999 \
    --add-qkv-bias \
    --disable-bias-linear \
    --attention-dropout 0.0 \
    --init-method-std 0.02 \
    --hidden-dropout 0.0 \
    --position-embedding-type rope \
    --normalization RMSNorm \
    --norm-epsilon 1e-06 \
    --swiglu \
    --use-flash-attn \
    --use-rotary-position-embeddings \
    --no-masked-softmax-fusion \
    --attention-softmax-in-fp32 \
    --initial-loss-scale 4096 \
    --no-gradient-accumulation-fusion \
    --no-load-optim \
    --no-load-rng \
    --seed 42 \
    --bf16\
    --tensorboard-dir $tensorboard_dir \
"

DATA_ARGS="
    --data-path $DATA_PATH \
    --split 100,0,0
"

OUTPUT_ARGS="
    --log-interval 1 \
    --save-interval 2000 \
    --eval-interval 2000 \
    --eval-iters 0 \
"

torchrun $DISTRIBUTED_ARGS posttrain_gpt.py \
    $GPT_ARGS \
    $DATA_ARGS \
    $OUTPUT_ARGS \
    --distributed-backend nccl \
    --load ${CKPT_LOAD_DIR} \
    --save ${CKPT_SAVE_DIR} \
    2>&1 | tee $train_log_path

cd ***/MindSpeed-LLM
bash /data/private/scripts/llm/qwen25-3b-instruct/tune_qwen25_3b_4k_full_ptd.sh