Qwen3-14B基于mindspeed-llm的训练适配

scripts/qwen3-14b#
    ├── ckpt_convert_qwen3_14b_hf2mcore.sh   # 将HF权重转换为Mcore格式(TP=4, PP=2)
    ├── ckpt_convert_qwen3_lora_merge.sh  # Lora微调Lora权重合并脚本
    ├── ckpt_convert_qwen3_mcore2hf_full.sh  # 将全参微调的Mcore权重转换为HF格式
    ├── ckpt_convert_qwen3_mcore2hf_lora.sh  # 将全参微调的Mcore权重转换为HF格式
    ├── data_convert_qwen3_instruction_customed10k_qwen3.sh # qwen3 customed慢思考数据转换脚本
    ├── data_convert_qwen3_pretrain_enwiki.sh # 预训练enwiki数据转换脚本
    ├── eval_qwen3_14b_full_customed10k.sh  # 全参微调的评测脚本
    ├── eval_qwen3_14b_lora_customed10k.sh  # Lora微调的评测脚本
    ├── pretrain_qwen3_14b_enwiki_4K_ptd.sh # 预训练训练脚本
    ├── tune_qwen3_14b_4K_full_customed10k_ptd_2nodes.sh # 全参微调双机训练脚本
    ├── tune_qwen3_14b_4K_full_customed10k_ptd.sh   # 全参微调训练脚本
    └── tune_qwen3_14b_4K_lora_customed10k_ptd.sh   # Lora微调训练脚本

models/Qwen3-14B# 
    ├── config.json
    ├── configuration.json
    ├── generation_config.json
    ├── merges.txt
    ├── mg2hf
    ├── model-00001-of-00008.safetensors
    ├── model-00002-of-00008.safetensors
    ├── model-00003-of-00008.safetensors
    ├── model-00004-of-00008.safetensors
    ├── model-00005-of-00008.safetensors
    ├── model-00006-of-00008.safetensors
    ├── model-00007-of-00008.safetensors
    ├── model-00008-of-00008.safetensors
    ├── model.safetensors.index.json
    ├── README.md
    ├── tokenizer_config.json
    ├── tokenizer.json
    └── vocab.json

datasets#
    ├── customed_test_1k.jsonl    # 测试集
    └── customed_train_10k.jsonl  # 训练集

OUTPUT_BASE_DIR=/data/qwen3
log_path="${OUTPUT_BASE_DIR}/14b/logs/ckpt_convert_qwen3_14b_hf2mcore.log"
mkdir -p ${OUTPUT_BASE_DIR}/14b/mg_weights/qwen3_14b_mcore_tp4pp2/
mkdir -p ${OUTPUT_BASE_DIR}/14b/logs

# 设置需要的权重转换参数
python convert_ckpt.py \
    --use-mcore-models \
    --model-type GPT \
    --load-model-type hf \
    --save-model-type mg \
    --target-tensor-parallel-size 4 \
    --target-pipeline-parallel-size 2 \
    --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
    --load-dir models/Qwen3-14B/ \
    --save-dir ${OUTPUT_BASE_DIR}/14b/mg_weights/qwen3_14b_mcore_tp4pp2/ \
    --tokenizer-model models/Qwen3-14B/tokenizer.json \
    --params-dtype bf16 \
    --model-type-hf qwen3 \
        | tee $log_path

cd scripts/qwen3-14b && bash ckpt_convert_qwen3_14b_hf2mcore.sh

/data/qwen3/14b/mg_weights/qwen3_14b_mcore_tp4pp2/
├── iter_0000001
│   ├── mp_rank_00_000
│   ├── mp_rank_00_001
│   ├── mp_rank_01_000
│   ├── mp_rank_01_001
│   ├── mp_rank_02_000
│   ├── mp_rank_02_001
│   ├── mp_rank_03_000
│   └── mp_rank_03_001
└── latest_checkpointed_iteration.txt

{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "请你判读用户的意图。...交互历史：用户：广州塔为什么也叫小蛮腰\n用户最后一句话所属的技能和意图是："}, {"role": "assistant", "content": "闲聊百科_知识查询"}]}

OUTPUT_BASE_DIR=/data/qwen3
mkdir -p ${OUTPUT_BASE_DIR}/finetune_dataset_customed10k_qwen3_think
log_path="${OUTPUT_BASE_DIR}/14b/logs/data_convert_qwen3_instruction_customed10k_qwen3.log"
mkdir -p ${OUTPUT_BASE_DIR}/14b/logs

python ./preprocess_data.py \
    --input  datasets/customed_train_10k.jsonl \
    --tokenizer-name-or-path models/Qwen3-14B/ \
    --output-prefix ${OUTPUT_BASE_DIR}/finetune_dataset_customed10k_qwen3_think/customed \
    --handler-name SharegptStyleInstructionHandler \
    --tokenizer-type PretrainedFromHF \
    --enable-thinking true \
    --workers 4 \
    --log-interval 1000 \
    --map-keys '{"messages":"messages", "tags":{"role_tag": "role","content_tag": "content","user_tag": "user","assistant_tag": "assistant","system_tag": "system"}}' \
    --prompt-type qwen3 \
        | tee $log_path

cd scripts/qwen3-14b && bash data_convert_qwen3_instruction_customed10k_qwen3.sh

.
├── customed_packed_attention_mask_document.bin
├── customed_packed_attention_mask_document.idx
├── customed_packed_input_ids_document.bin
├── customed_packed_input_ids_document.idx
├── customed_packed_labels_document.bin
├── customed_packed_labels_document.idx
├── customed_train_indexmap_49920ns_42s_shuffle_decoder_packed_idx.npy
└── customed_train_indexmap_51200ns_42s_shuffle_decoder_packed_idx.npy

{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "请你判读用户的意图。...交互历史：用户：广州塔为什么也叫小蛮腰\n用户最后一句话所属的技能和意图是："}, {"role": "assistant", "content": "闲聊百科_知识查询"}]}

{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "请你判读用户的意图。...交互历史：用户：广州塔为什么也叫小蛮腰\n用户最后一句话所属的技能和意图是："}, {"role": "assistant", "content": "闲聊百科_知识查询"}]}

{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "请你判读用户的意图。...交互历史：用户：广州塔为什么也叫小蛮腰\n用户最后一句话所属的技能和意图是："}, {"role": "assistant", "content": "闲聊百科_知识查询"}]}
{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "请你判读用户的意图。...交互历史：用户：广州塔为什么也叫小蛮腰\n用户最后一句话所属的技能和意图是："}, {"role": "assistant", "content": "闲聊百科_知识查询"}]}

export CUDA_DEVICE_MAX_CONNECTIONS=1
#基础配置
NPUS_PER_NODE=16  #使用单节点的NPU
MASTER_ADDR=localhost #以本节点ip地址为master_ip
MASTER_PORT=6015 #本节点端口号为6015
NNODES=1  #单机，即一台节点，多机即多节点
NODE_RANK=0  #单机RANK为0，多机为(0,NNODES-1)，不同节点不可重复
WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES)) #最终使用的NPU数量

OUTPUT_BASE_DIR=/data/qwen3
mkdir -p ${OUTPUT_BASE_DIR}/14b/logs

# 根据实际情况配置权重保存、权重加载、词表、数据集路径
CKPT_LOAD_DIR="${OUTPUT_BASE_DIR}/14b/mg_weights/qwen3_14b_mcore_tp4pp2/"  #权重加载路径，填入权重转换时保存的权重路径
CKPT_SAVE_DIR="${OUTPUT_BASE_DIR}/14b/save_weights_full/qwen3_14b_mcore_tp4pp2_customed10k_qwen3_think/"  #训练完成后的权重保存路径
DATA_PATH="${OUTPUT_BASE_DIR}/finetune_dataset_customed10k_qwen3_think/customed"  #数据集路径，填入数据预处理时保存的数据路径，注意需要添加后缀
TOKENIZER_PATH="models/Qwen3-14B/" #词表路径，填入下载的开源权重词表路径
log_path="${OUTPUT_BASE_DIR}/14b/logs/tune_qwen3_14b_4K_full_customed10k_ptd.log"

TP=4  # 张量并行维度
PP=2  # 流水线并行维度
MBS=16  # 微批次大小（每NPU单次前向计算样本数）
GBS=128 # 全局批次大小
SEQ_LENGTH=800   # 最大序列长度（模型支持4096上下文）
TRAIN_ITERS=390 #训练步数

DISTRIBUTED_ARGS="
    --nproc_per_node $NPUS_PER_NODE \ 
    --nnodes $NNODES \
    --node_rank $NODE_RANK \
    --master_addr $MASTER_ADDR \
    --master_port $MASTER_PORT
"

GPT_ARGS="
    --use-mcore-models \
    --tensor-model-parallel-size ${TP} \
    --pipeline-model-parallel-size ${PP} \
    --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
    --kv-channels 128 \
    --qk-layernorm \
    --num-layers 40 \
    --hidden-size 5120 \
    --use-rotary-position-embeddings \
    --untie-embeddings-and-output-weights \
    --num-attention-heads 40 \
    --ffn-hidden-size 17408 \
    --max-position-embeddings 40960 \
    --seq-length ${SEQ_LENGTH} \
    --train-iters ${TRAIN_ITERS} \
    --micro-batch-size ${MBS} \
    --global-batch-size ${GBS} \
    --make-vocab-size-divisible-by 1 \
    --use-flash-attn \
    --padded-vocab-size 151936 \
    --rotary-base 1000000 \
    --disable-bias-linear \
    --swiglu \
    --tokenizer-type PretrainedFromHF \
    --tokenizer-name-or-path ${TOKENIZER_PATH} \
    --normalization RMSNorm \
    --position-embedding-type rope \
    --norm-epsilon 1e-6 \
    --hidden-dropout 0 \
    --attention-dropout 0 \
    --no-gradient-accumulation-fusion \
    --attention-softmax-in-fp32 \
    --exit-on-missing-checkpoint \
    --no-masked-softmax-fusion \
    --group-query-attention \
    --num-query-groups 8 \
    --seed 42 \
    --bf16 \
    --lr-warmup-fraction 0.1 \
    --clip-grad 1.0 \
    --adam-beta1 0.9 \
    --adam-beta2 0.999 \
    --no-load-optim \
    --no-load-rng \
    --lr 1.0e-5 \
    --sequence-parallel
"

DATA_ARGS="
    --data-path $DATA_PATH \
    --split 100,0,0
"

OUTPUT_ARGS="
    --log-interval 1 \
    --save-interval ${TRAIN_ITERS} \
    --eval-interval ${TRAIN_ITERS} \
    --eval-iters 0 \
    --log-throughput
"

TUNE_ARGS="
    --finetune \
    --stage sft \
    --is-instruction-dataset \
    --prompt-type qwen \
    --variable-seq-lengths
"

torchrun $DISTRIBUTED_ARGS posttrain_gpt.py \
    $GPT_ARGS \
    $DATA_ARGS \
    $OUTPUT_ARGS \
    $TUNE_ARGS \
    --tensorboard-dir ${OUTPUT_BASE_DIR}/14b/tb/full/customed10k \
    --distributed-backend nccl \
    --load ${CKPT_LOAD_DIR} \
    --save ${CKPT_SAVE_DIR} \
    | tee $log_path

LLaMAFactory+GPU超参：
### model
model_name_or_path:  XXXXXX
trust_remote_code: true

### method
stage: sft
do_train: true
finetuning_type: full
deepspeed: examples/deepspeed/ds_z3_config.json

### dataset
dataset: customed
template: qwen3
cutoff_len: 800
max_samples: 10000
overwrite_cache: true
preprocessing_num_workers: 16
dataloader_num_workers: 4

output_dir: XXXXXX
logging_steps: 10
save_steps: 100
plot_loss: true
overwrite_output_dir: true
save_only_model: false
report_to: tensorboard  # choices: [none, wandb, tensorboard, swanlab, mlflow]

### train
per_device_train_batch_size: 2
gradient_accumulation_steps: 8
learning_rate: 1.0e-5
num_train_epochs: 5
lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: true
ddp_timeout: 180000000
resume_from_checkpoint: null

### eval
# eval_dataset: alpaca_en_demo
# val_size: 0.1
# per_device_eval_batch_size: 1
# eval_strategy: steps
# eval_steps: 500

cd scripts/qwen3-14b && bash tune_qwen3_14b_4K_full_customed10k_ptd.sh

tensorboard --logdir=/data/qwen3/14b/tb/full/customed10k --port=3000

# 基础配置
NPUS_PER_NODE=16  #使用单节点的8卡NPU
MASTER_ADDR=$(cat /etc/volcano/worker.host | head -n 1 | awk '{print $1}') #以本节点ip地址为master_ip
MASTER_PORT=6016 #本节点端口号为6015
NNODES=${VC_WORKER_NUM}  #单机，即一台节点，多机即多节点
NODE_RANK=${VC_TASK_INDEX}  #单机RANK为0，多机为(0,NNODES-1)，不同节点不可重复
WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES)) #最终使用的NPU数

cd scripts/qwen3-14b && bash tune_qwen3_14b_4K_full_customed10k_ptd_2nodes.sh

tensorboard --logdir=/data/qwen3/14b/tb --port=3000

export CUDA_DEVICE_MAX_CONNECTIONS=1

OUTPUT_BASE_DIR=/data/qwen3
CHECKPOINT="${OUTPUT_BASE_DIR}/14b/save_weights_full/qwen3_14b_mcore_tp4pp2_customed10k_qwen3_think/" # 指向微调后权重的保存路径
TOKENIZER_PATH="models/Qwen3-14B/" # 指向模型tokenizer的路径
eval_log_path="${OUTPUT_BASE_DIR}/14b/logs/eval_qwen3_14b_full_customed10k.log"
EVAL_DATA_PATH="datasets/customed_test_1k.jsonl"

# Change for multinode config
MASTER_ADDR=localhost
MASTER_PORT=6000
NNODES=1 # 集群里的节点数，以实际情况填写,
NODE_RANK=0  # 当前节点的RANK，多个节点不能重复，主节点为0, 其他节点可以是1,2..
NPUS_PER_NODE=8
WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))

TP=4  # 张量并行维度
PP=2  # 流水线并行维度
SEQ_LENGTH=800 # 最大序列长度

DISTRIBUTED_ARGS="
    --nproc_per_node $NPUS_PER_NODE \
    --nnodes $NNODES \
    --node_rank $NODE_RANK \
    --master_addr $MASTER_ADDR \
    --master_port $MASTER_PORT
"

torchrun $DISTRIBUTED_ARGS inference_4_midea_test.py \
         --use-mcore-models \
         --tensor-model-parallel-size ${TP} \
         --pipeline-model-parallel-size ${PP} \
         --load ${CHECKPOINT} \
         --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
         --kv-channels 128 \
         --qk-layernorm \
         --num-layers 40 \
         --hidden-size 5120 \
         --use-rotary-position-embeddings \
         --untie-embeddings-and-output-weights \
         --num-attention-heads 40 \
         --ffn-hidden-size 17408 \
         --max-position-embeddings 40960 \
         --seq-length ${SEQ_LENGTH} \
         --make-vocab-size-divisible-by 1 \
         --padded-vocab-size 151936 \
         --rotary-base 1000000 \
         --micro-batch-size 1 \
         --disable-bias-linear \
         --swiglu \
         --use-rotary-position-embeddings \
         --tokenizer-type PretrainedFromHF \
         --tokenizer-name-or-path ${TOKENIZER_PATH} \
         --normalization RMSNorm \
         --position-embedding-type rope \
         --norm-epsilon 1e-6 \
         --hidden-dropout 0 \
         --attention-dropout 0 \
         --max-new-tokens 256 \
         --no-gradient-accumulation-fusion \
         --attention-softmax-in-fp32 \
         --exit-on-missing-checkpoint \
         --no-masked-softmax-fusion \
         --group-query-attention \
         --num-query-groups 8 \
         --seed 42 \
         --bf16 \
      	 --eval-data-path ${EVAL_DATA_PATH} \
       	 --eval-data-size 1000 \
	     --temperature 0.00001 \
       		| tee $eval_log_path

cd scripts/qwen3-14b && bash eval_qwen3_14b_full_customed10k.sh

export CUDA_DEVICE_MAX_CONNECTIONS=1

OUTPUT_BASE_DIR=/data/qwen3
log_path="${OUTPUT_BASE_DIR}/14b/logs/ckpt_convert_qwen3_mcore2hf_full.log"
mkdir -p ${OUTPUT_BASE_DIR}/14b/logs

python convert_ckpt.py \
    --use-mcore-models \
    --model-type GPT \
    --load-model-type mg \
    --save-model-type hf \
    --target-tensor-parallel-size 1 \
    --target-pipeline-parallel-size 1 \
    --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
    --load-dir ${OUTPUT_BASE_DIR}/14b/save_weights_full/qwen3_14b_mcore_tp4pp2_customed10k_qwen3_think/ \
    --save-dir models/Qwen3-14B/ \
    --params-dtype bf16 \
    --model-type-hf qwen3 \
        | tee $log_path

cp -rf models/Qwen3-14B/mg2hf ${OUTPUT_BASE_DIR}/14b/save_weights_full/

cd scripts/qwen3-14b && bash ckpt_convert_qwen3_mcore2hf_full.sh

/data/qwen3//14b/save_weights_full/mg2hf/
├── config.json
├── generation_config.json
├── model-00001-of-00006.safetensors
├── model-00002-of-00006.safetensors
├── model-00003-of-00006.safetensors
├── model-00004-of-00006.safetensors
├── model-00005-of-00006.safetensors
├── model-00006-of-00006.safetensors
└── model.safetensors.index.json

export CUDA_DEVICE_MAX_CONNECTIONS=1
# 基础配置
NPUS_PER_NODE=8  #使用单节点的8卡NPU
MASTER_ADDR=localhost #以本节点ip地址为master_ip
MASTER_PORT=6015 #本节点端口号为6015
NNODES=1  #单机，即一台节点，多机即多节点
NODE_RANK=0  #单机RANK为0，多机为(0,NNODES-1)，不同节点不可重复
WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES)) #最终使用的NPU数

OUTPUT_BASE_DIR=/data/qwen3
mkdir -p ${OUTPUT_BASE_DIR}/14b/logs

# 根据实际情况配置权重保存、权重加载、词表、数据集路径
CKPT_LOAD_DIR="${OUTPUT_BASE_DIR}/14b/mg_weights/qwen3_14b_mcore_tp4pp2/"  #权重加载路径，填入权重转换时保存的权重路径
CKPT_SAVE_DIR="${OUTPUT_BASE_DIR}/14b/save_weights_lora/qwen3_14b_mcore_tp4pp2_customed10k_qwen3_think/"  #训练完成后的权重保存路径
DATA_PATH="${OUTPUT_BASE_DIR}/finetune_dataset_customed10k_qwen3_think/customed"  #数据集路径，填入数据预处理时保存的数据路径，注意需要添加后缀
TOKENIZER_PATH="models/Qwen3-14B/" #词表路径，填入下载的开源权重词表路径
log_path="${OUTPUT_BASE_DIR}/14b/logs/tune_qwen3_14b_4K_lora_customed10k_ptd.log"
TP=4  # 张量并行维度
PP=2  # 流水线并行维度
SEQ_LENGTH=2048  # 最大序列长度
TRAIN_ITERS=790  #训练步数
MBS=1    # 微批次大小（每NPU单次前向计算样本数）
GBS=8    # 全局批次大小

DISTRIBUTED_ARGS="
    --nproc_per_node $NPUS_PER_NODE \
    --nnodes $NNODES \
    --node_rank $NODE_RANK \
    --master_addr $MASTER_ADDR \
    --master_port $MASTER_PORT
"

OPTIMIZE_ARGS="
    --use-flash-attn \
    --use-fused-rotary-pos-emb \
    --use-rotary-position-embeddings \
    --use-fused-swiglu \
    --use-fused-rmsnorm \
    --no-masked-softmax-fusion \
    --use-distributed-optimizer
"

TRAIN_ARGS="
    --micro-batch-size ${MBS} \
    --global-batch-size ${GBS} \
    --lr 1.0e-5 \
    --lr-decay-style cosine \
    --lr-warmup-fraction 0.1 \
    --attention-dropout 0.0 \
    --init-method-std 0.01 \
    --hidden-dropout 0.0 \
    --clip-grad 1.0 \
    --adam-beta1 0.9 \
    --adam-beta2 0.999 \
    --initial-loss-scale 4096 \
    --seed 42 \
    --bf16 \
    --train-iters ${TRAIN_ITERS} \
    --seq-length ${SEQ_LENGTH} \
    --no-shared-storage
"

MODEL_PARALLEL_ARGS="
    --tensor-model-parallel-size ${TP} \
    --pipeline-model-parallel-size ${PP}
"

GPT_ARGS="
    --use-mcore-models \
    --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
    --kv-channels 128 \
    --qk-layernorm \
    --tokenizer-name-or-path ${TOKENIZER_PATH} \
    --max-position-embeddings ${SEQ_LENGTH} \
    --num-layers 40 \
    --hidden-size 5120 \
    --ffn-hidden-size 17408 \
    --num-attention-heads 40 \
    --tokenizer-type PretrainedFromHF \
    --make-vocab-size-divisible-by 1 \
    --padded-vocab-size 151936 \
    --rotary-base 1000000 \
    --untie-embeddings-and-output-weights \
    --disable-bias-linear \
    --position-embedding-type rope \
    --normalization RMSNorm \
    --swiglu \
    --attention-softmax-in-fp32 \
    --no-gradient-accumulation-fusion \
    --group-query-attention \
    --num-query-groups 8
"

DATA_ARGS="
    --data-path $DATA_PATH \
    --split 100,0,0
"

OUTPUT_ARGS="
    --load ${CKPT_LOAD_DIR} \
    --save ${CKPT_SAVE_DIR} \
    --log-interval 1 \
    --save-interval ${TRAIN_ITERS} \
    --eval-interval ${TRAIN_ITERS} \
    --eval-iters 0 \
    --no-load-optim \
    --no-load-rng
"

TUNE_ARGS="
    --finetune \
    --stage sft \
    --is-instruction-dataset \
    --tokenizer-not-use-fast \
    --prompt-type qwen \
    --variable-seq-lengths \
    --lora-r 16 \
    --lora-alpha 32 \
    --lora-fusion \
    --lora-target-modules linear_qkv linear_proj linear_fc1 linear_fc2
"

torchrun $DISTRIBUTED_ARGS posttrain_gpt.py \
    $GPT_ARGS \
    $DATA_ARGS \
    $OUTPUT_ARGS \
    $OPTIMIZE_ARGS \
    $TRAIN_ARGS \
    $TUNE_ARGS \
    $MODEL_PARALLEL_ARGS \
    --tensorboard-dir ${OUTPUT_BASE_DIR}/14b/tb/lora/customed10k \
    --distributed-backend nccl \
    | tee $log_path

LLaMAFactory+GPU超参：
### model
model_name_or_path: XXXXXX
trust_remote_code: true

### method
stage: sft
do_train: true
finetuning_type: lora
lora_rank: 16
lora_target: all

### dataset
dataset: customed
template: qwen3
cutoff_len: 2048
max_samples: 10000
overwrite_cache: true
preprocessing_num_workers: 16
dataloader_num_workers: 4

### output
output_dir: XXXXXX
logging_steps: 10
save_steps: 100
plot_loss: true
overwrite_output_dir: true
save_only_model: false
report_to: tensorboard  # choices: [none, wandb, tensorboard, swanlab, mlflow]

### train
per_device_train_batch_size: 4
gradient_accumulation_steps: 4
learning_rate: 1.0e-5
num_train_epochs: 5.0
lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: true
ddp_timeout: 180000000
resume_from_checkpoint: null

### eval
# eval_dataset: alpaca_en_demo
# val_size: 0.1
# per_device_eval_batch_size: 1
# eval_strategy: steps
# eval_steps: 500

cd scripts/qwen3-14b && bash tune_qwen3_14b_4K_lora_customed10k_ptd.sh

tensorboard --logdir=/data/qwen3/14b/tb/full/customed10k --port=3000

export CUDA_DEVICE_MAX_CONNECTIONS=1

OUTPUT_BASE_DIR=/data/qwen3
CHECKPOINT="${OUTPUT_BASE_DIR}/14b/save_weights_lora/qwen3_14b_mcore_tp4pp2_customed10k_qwen3_think_merge/" # 指向微调后权重的保存路径
TOKENIZER_PATH="models/Qwen3-14B/" # 指向模型tokenizer的路径
eval_log_path="${OUTPUT_BASE_DIR}/14b/logs/eval_qwen3_14b_lora_customed10k.log"
EVAL_DATA_PATH="datasets/customed_test_1k.jsonl"

# Change for multinode config
MASTER_ADDR=localhost
MASTER_PORT=6000
NNODES=1 # 集群里的节点数，以实际情况填写,
NODE_RANK=0  # 当前节点的RANK，多个节点不能重复，主节点为0, 其他节点可以是1,2..
NPUS_PER_NODE=8  # 单节点NPU芯片数量
WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES)) # 全局NPU总数=单节点NPU数×节点数

TP=4
PP=2
SEQ_LENGTH=2048

DISTRIBUTED_ARGS="
    --nproc_per_node $NPUS_PER_NODE \
    --nnodes $NNODES \
    --node_rank $NODE_RANK \
    --master_addr $MASTER_ADDR \
    --master_port $MASTER_PORT
"

torchrun $DISTRIBUTED_ARGS inference_4_midea_test.py \
         --use-mcore-models \
         --tensor-model-parallel-size ${TP} \
         --pipeline-model-parallel-size ${PP} \
         --load ${CHECKPOINT} \
         --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
         --kv-channels 128 \
         --qk-layernorm \
         --num-layers 40 \
         --hidden-size 5120 \
         --use-rotary-position-embeddings \
         --untie-embeddings-and-output-weights \
         --num-attention-heads 40 \
         --ffn-hidden-size 17408 \
         --max-position-embeddings 40960 \
         --seq-length ${SEQ_LENGTH} \
         --make-vocab-size-divisible-by 1 \
         --padded-vocab-size 151936 \
         --rotary-base 1000000 \
         --micro-batch-size 1 \
         --disable-bias-linear \
         --swiglu \
         --use-rotary-position-embeddings \
         --tokenizer-type PretrainedFromHF \
         --tokenizer-name-or-path ${TOKENIZER_PATH} \
         --normalization RMSNorm \
         --position-embedding-type rope \
         --norm-epsilon 1e-6 \
         --hidden-dropout 0 \
         --attention-dropout 0 \
         --max-new-tokens 256 \
         --no-gradient-accumulation-fusion \
         --attention-softmax-in-fp32 \
         --exit-on-missing-checkpoint \
         --no-masked-softmax-fusion \
         --group-query-attention \
         --num-query-groups 8 \
         --seed 42 \
      	 --eval-data-path ${EVAL_DATA_PATH} \
       	 --eval-data-size 1000 \
  	     --temperature 0.00001 \
                | tee $eval_log_path

cd scripts/qwen3-14b && bash eval_qwen3_14b_lora_customed10k.sh

export CUDA_DEVICE_MAX_CONNECTIONS=1

OUTPUT_BASE_DIR=/data/qwen3
log_path="${OUTPUT_BASE_DIR}/14b/logs/ckpt_convert_qwen3_mcore2hf_lora.log"
mkdir -p ${OUTPUT_BASE_DIR}/14b/logs

python convert_ckpt.py \
    --use-mcore-models \
    --model-type GPT \
    --load-model-type mg \
    --save-model-type hf \
    --target-tensor-parallel-size 1 \
    --target-pipeline-parallel-size 1 \
    --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
    --load-dir  ${OUTPUT_BASE_DIR}/14b/save_weights_lora/qwen3_14b_mcore_tp4pp2_customed10k_qwen3_think_merge/ \
    --save-dir models/Qwen3-14B_lora/ \
    --model-type-hf qwen3 \
        | tee $log_path

cp -rf models/Qwen3-14B_lora/mg2hf ${OUTPUT_BASE_DIR}/14b/save_weights_lora/

cd scripts/qwen3-14b && bash ckpt_convert_qwen3_mcore2hf_lora.sh

/data/qwen3//14b/save_weights_lora/mg2hf/
├── config.json
├── generation_config.json
├── model-00001-of-00006.safetensors
├── model-00002-of-00006.safetensors
├── model-00003-of-00006.safetensors
├── model-00004-of-00006.safetensors
├── model-00005-of-00006.safetensors
├── model-00006-of-00006.safetensors
└── model.safetensors.index.json

OUTPUT_BASE_DIR=/data/qwen3
mkdir -p ${OUTPUT_BASE_DIR}/pretrain_datasets_enwiki
log_path="${OUTPUT_BASE_DIR}/14b/logs/data_convert_qwen3_pretrain_enwiki.log"

python ./preprocess_data.py \
    --input datasets/enwiki/train-00000-of-00042-d964455e17e96d5a.parquet \
    --tokenizer-name-or-path models/Qwen3-14B/ \
    --tokenizer-type PretrainedFromHF \
    --handler-name GeneralPretrainHandler \
    --output-prefix ${OUTPUT_BASE_DIR}/pretrain_datasets_enwiki/enwiki \
    --json-keys text \
    --workers 4 \
    --log-interval 1000 \
        | tee $log_path

cd scripts/qwen3-14b && bash data_convert_qwen3_pretrain_enwiki.sh

/data/qwen3/pretrain_datasets_enwiki/
├── enwiki_text_document
│   └── cache
├── enwiki_text_document.bin
└── enwiki_text_document.idx

export CUDA_DEVICE_MAX_CONNECTIONS=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
# 基础配置
NPUS_PER_NODE=16  #A3单节点16卡NPU
MASTER_ADDR=localhost #以本节点ip地址为master_ip
MASTER_PORT=6115 #本节点端口号为6015
NNODES=1  #单机，即一台节点，多机即多节点
NODE_RANK=0  #单机RANK为0，多机为(0,NNODES-1)，不同节点不可重复
WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES)) #最终使用的NPU数

OUTPUT_BASE_DIR=/data/qwen3

# 根据实际情况配置权重保存、权重加载、词表、数据集路径
CKPT_LOAD_DIR="${OUTPUT_BASE_DIR}/14b/mg_weights/qwen3_14b_mcore_tp4pp2/"  # 权重加载路径，填入权重转换时保存的权重路径
CKPT_SAVE_DIR="${OUTPUT_BASE_DIR}/14b/save_weights_pretrain/qwen3_14b_mcore_tp4pp2/"                # 训练完成后的权重保存路径
DATA_PATH="${OUTPUT_BASE_DIR}/pretrain_datasets_enwiki/enwiki_text_document"      # 数据集路径，填入数据预处理时保存的数据路径，注意需要添加后缀
TOKENIZER_PATH="models/Qwen3-14B/" # 词表路径，填入下载的开源权重词表路径
log_path="${OUTPUT_BASE_DIR}/14b/logs/pretrain_qwen3_14b_enwiki_4K_ptd.log"

TP=4  
PP=2 
CP=1  #序列并行
MBS=4 #设置micro-batch-size为4
GBS=64 #设置global-batch-size为128
SEQ_LENGTH=4096  #设置seq_length为4096 
TRAIN_ITERS=2000 
CP_TYPE='ulysses_cp_algo' #序列并行算法

DISTRIBUTED_ARGS="
    --nproc_per_node $NPUS_PER_NODE \
    --nnodes $NNODES \
    --node_rank $NODE_RANK \
    --master_addr $MASTER_ADDR \
    --master_port $MASTER_PORT
"

OPTIMIZE_ARGS="
    --use-flash-attn \
    --use-fused-rotary-pos-emb \
    --use-rotary-position-embeddings \
    --use-fused-swiglu \
    --use-fused-rmsnorm \
    --no-masked-softmax-fusion \
    --use-distributed-optimizer
"

TRAIN_ARGS="
    --micro-batch-size ${MBS} \
    --global-batch-size ${GBS} \
    --lr 1.25e-6 \
    --min-lr 1.25e-7 \
    --weight-decay 1e-1 \
    --lr-warmup-fraction 0.01 \
    --attention-dropout 0.0 \
    --hidden-dropout 0.0 \
    --clip-grad 1.0 \
    --adam-beta1 0.9 \
    --adam-beta2 0.95 \
    --seed 42 \
    --bf16 \
    --train-iters ${TRAIN_ITERS} \
    --seq-length ${SEQ_LENGTH} \
"

MODEL_PARALLEL_ARGS="
    --tensor-model-parallel-size ${TP} \
    --pipeline-model-parallel-size ${PP} \
    --context-parallel-size ${CP} \
    --context-parallel-algo ${CP_TYPE} \
"

GPT_ARGS="
    --use-mcore-models \
    --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
    --kv-channels 128 \
    --qk-layernorm \
    --num-layers 40 \
    --hidden-size 5120 \
    --untie-embeddings-and-output-weights \
    --num-attention-heads 40 \
    --ffn-hidden-size 17408 \
    --max-position-embeddings 40960 \
    --make-vocab-size-divisible-by 1 \
    --padded-vocab-size 151936 \
    --rotary-base 1000000 \
    --disable-bias-linear \
    --swiglu \
    --tokenizer-type PretrainedFromHF \
    --tokenizer-name-or-path ${TOKENIZER_PATH} \
    --normalization RMSNorm \
    --position-embedding-type rope \
    --norm-epsilon 1e-6 \
    --no-gradient-accumulation-fusion \
    --attention-softmax-in-fp32 \
    --exit-on-missing-checkpoint \
    --group-query-attention \
    --num-query-groups 8 \
    --no-load-optim \
    --no-load-rng \
    --tensorboard-dir MindSpeed-LLM/tb \
    --sequence-parallel
"

DATA_ARGS="
    --data-path $DATA_PATH \
    --split 100,0,0
"

OUTPUT_ARGS="
    --log-interval 1 \
    --save-interval 20000 \
    --eval-interval ${TRAIN_ITERS} \
    --eval-iters 0 \
    --log-throughput
"

torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \
    $GPT_ARGS \
    $DATA_ARGS \
    $OUTPUT_ARGS \
    $OPTIMIZE_ARGS \
    $TRAIN_ARGS \
    $MODEL_PARALLEL_ARGS \
    --distributed-backend nccl \
    --tensorboard-dir ${OUTPUT_BASE_DIR}/14b/tb/pretrain/enwiki \
    --load ${CKPT_LOAD_DIR} \
    --save ${CKPT_SAVE_DIR} \
    | tee $log_path

cd scripts/qwen3-14b && bash pretrain_qwen3_14b_enwiki_4K_ptd.sh

tensorboard --logdir=/data/qwen3/14b/tb/full/customed10k --port=3000