chmod +x Ascend-cann-*.run 
./$(ls -t Ascend-cann-toolkit*.run|head -n 1) --install --quiet
./$(ls -t Ascend-cann-kernels*.run|head -n 1) --install --quiet
source /usr/local/Ascend/ascend-toolkit/set_env.sh
./$(ls -t Ascend-cann-nnal*.run|head -n 1) --install --quiet
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh

pip install --upgrade pip 
pip install $(ls -t /tmp/torch*.whl|grep -v "torch_npu"|head -n 1)
pip install $(ls -t /tmp/torch_npu*.whl|head -n 1)

source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh

# 安装MindSpeed加速库
git clone https://gitee.com/ascend/MindSpeed.git
cd MindSpeed
git checkout 2c085cc9  # checkout commit from MindSpeed core_r0.8.0 in 2025.04.01
pip install -r requirements.txt 
pip3 install -e .
cd ..

# 准备MindSpeed-LLM及Megatron-LM源码
git clone https://gitee.com/ascend/MindSpeed-LLM.git 
# megatron从github下载，请确保网络能访问
git clone https://github.com/NVIDIA/Megatron-LM.git  
cd Megatron-LM
git checkout core_r0.8.0
cp -r megatron ../MindSpeed-LLM/
cd ../MindSpeed-LLM
git checkout 2.1.0

pip install -r requirements.txt  # 安装其余依赖库

# add ma user group if not exists
groupadd ma-group -g 100

# add and chown ma user if not exists
useradd -m -d /home/ma-user -s /bin/bash -g 100 -u 1000 ma-user

# add permission
chown -R ma-user:100 /usr/local/*

ARG BASE_IMAGE=openeuler-22.03-lts-sp3:latest
FROM ${BASE_IMAGE}

# Install basic software
RUN yum update -y && \
    yum install -y \
        gcc \
        gcc-c++ \
        make \
        cmake \
        curl \
        zlib-devel \
        bzip2-devel \
        openssl-devel \
        ncurses-devel \
        sqlite-devel \
        readline-devel \
        tk-devel \
        gdbm-devel \
        libpcap-devel \
        xz-devel \
        libev-devel \
        expat-devel \
        libffi-devel \
        systemtap-sdt-devel \
        unzip \
        pciutils \
        net-tools \
        lapack-devel \
        gcc-gfortran \
        util-linux \
        findutils \
        wget \
        ca-certificates \
        bash \
        glibc \
        git \
        patch \
    && yum clean all \
    && rm -rf /var/cache/yum \
    && rm -rf /tmp/*

# Install Python
ENV PATH=/usr/local/python3/bin:${PATH}
ADD ./Python-3* /tmp
RUN cd /tmp/Python-3* && \
    mkdir -p /usr/local/python3/lib && \
    ./configure --enable-shared --enable-shared LDFLAGS="-Wl,-rpath /usr/local/python3/lib" --prefix=/usr/local/python3 && \
    make -j $(nproc) && \
    make altinstall && \
    ln -sf $(ls -t /usr/local/python3/bin/python3.*[0-9]|head -n 1) /usr/local/python3/bin/python3 && \
    ln -sf $(ls -t /usr/local/python3/bin/pip3.*[0-9]|head -n 1) /usr/local/python3/bin/pip3 && \
    ln -sf /usr/local/python3/bin/python3 /usr/local/python3/bin/python && \
    ln -sf /usr/local/python3/bin/pip3 /usr/local/python3/bin/pip && \
    rm -rf /tmp/* 

# Stage 2: Install CANN
# Note: Install CANN runtime dependencies
RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/ && pip config set global.timeout 6000
RUN pip install --no-cache-dir --upgrade pip && \
    pip install --no-cache-dir \
        attrs \
        cython \
        numpy \
        decorator \
        sympy \
        cffi \
        pyyaml \
        pathlib2 \
        psutil \
        protobuf \
        scipy \
        requests \
        absl-py

COPY ./Ascend-cann*.run /tmp/
RUN cd /tmp && if (ls Ascend-cann*.run 1>/dev/null 2>&1); then \
        ls && chmod +x Ascend-cann-*.run \
                && ./$(ls -t Ascend-cann-toolkit*.run|head -n 1) --quiet --install --install-for-all \
                && ./$(ls -t Ascend-cann-kernels*.run|head -n 1)  --quiet --install --install-for-all \
                && source /usr/local/Ascend/ascend-toolkit/set_env.sh \
                && ./$(ls -t Ascend-cann-nnal*.run|head -n 1) --quiet --install --install-for-all \
                && source /usr/local/Ascend/ascend-toolkit/set_env.sh \
                && source /usr/local/Ascend/nnal/atb/set_env.sh; \
    else \
        echo "CANN install package does not exist, skip."; \
    fi \
    && rm -f /tmp/Ascend-cann*.run

# Note: Toolkit Environment variables, obtained from /usr/local/Ascend/ascend-toolkit/set_env.sh
ENV ASCEND_TOOLKIT_HOME=/usr/local/Ascend/ascend-toolkit/latest
ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:$LD_LIBRARY_PATH
ENV LD_LIBRARY_PATH=${ASCEND_TOOLKIT_HOME}/lib64:${ASCEND_TOOLKIT_HOME}/lib64/plugin/opskernel:${ASCEND_TOOLKIT_HOME}/lib64/plugin/nnengine:${ASCEND_TOOLKIT_HOME}/opp/built-in/op_impl/ai_core/tbe/op_tiling:$LD_LIBRARY_PATH
ENV LD_LIBRARY_PATH=${ASCEND_TOOLKIT_HOME}/tools/aml/lib64:${ASCEND_TOOLKIT_HOME}/tools/aml/lib64/plugin:$LD_LIBRARY_PATH
ENV PYTHONPATH=${ASCEND_TOOLKIT_HOME}/python/site-packages:${ASCEND_TOOLKIT_HOME}/opp/built-in/op_impl/ai_core/tbe:$PYTHONPATH
ENV PATH=${ASCEND_TOOLKIT_HOME}/bin:${ASCEND_TOOLKIT_HOME}/compiler/ccec_compiler/bin:${ASCEND_TOOLKIT_HOME}/tools/ccec_compiler/bin:$PATH
ENV ASCEND_AICPU_PATH=${ASCEND_TOOLKIT_HOME}
ENV ASCEND_OPP_PATH=${ASCEND_TOOLKIT_HOME}/opp
ENV TOOLCHAIN_HOME=${ASCEND_TOOLKIT_HOME}/toolkit
ENV ASCEND_HOME_PATH=${ASCEND_TOOLKIT_HOME}

# Note: NNAL Environment variables, obtained from /usr/local/Ascend/nnal/set_env.sh
ENV ATB_HOME_PATH=/usr/local/Ascend/nnal/atb/latest/atb/cxx_abi_0
ENV LD_LIBRARY_PATH=${ATB_HOME_PATH}/lib:${ATB_HOME_PATH}/examples:${ATB_HOME_PATH}/tests/atbopstest:${LD_LIBRARY_PATH}
ENV PATH=${ATB_HOME_PATH}/bin:$PATH
ENV ATB_STREAM_SYNC_EVERY_KERNEL_ENABLE=0
ENV ATB_STREAM_SYNC_EVERY_RUNNER_ENABLE=0
ENV ATB_STREAM_SYNC_EVERY_OPERATION_ENABLE=0
ENV ATB_OPSRUNNER_SETUP_CACHE_ENABLE=1
ENV ATB_OPSRUNNER_KERNEL_CACHE_LOCAL_COUNT=1
ENV ATB_OPSRUNNER_KERNEL_CACHE_GLOABL_COUNT=5
ENV ATB_WORKSPACE_MEM_ALLOC_ALG_TYPE=1
ENV ATB_WORKSPACE_MEM_ALLOC_GLOBAL=0
ENV ATB_COMPARE_TILING_EVERY_KERNEL=0
ENV ATB_HOST_TILING_BUFFER_BLOCK_NUM=128
ENV ATB_DEVICE_TILING_BUFFER_BLOCK_NUM=32
ENV ATB_SHARE_MEMORY_NAME_SUFFIX=""
ENV ATB_MATMUL_SHUFFLE_K_ENABLE=1
ENV ASDOPS_LOG_LEVEL=ERROR
ENV ASDOPS_LOG_TO_STDOUT=0
ENV ASDOPS_LOG_TO_FILE=1
ENV ASDOPS_LOG_TO_FILE_FLUSH=0
ENV ASDOPS_LOG_TO_BOOST_TYPE=atb
ENV ASDOPS_LOG_PATH=/root
ENV LCCL_DETERMINISTIC=0
ENV LCCL_PARALLEL=0
ENV PYTHONUNBUFFERED=1

SHELL [ "/bin/bash", "-c" ]

# Note: Set environment variables
RUN CANN_TOOLKIT_ENV_FILE="/usr/local/Ascend/ascend-toolkit/set_env.sh" && \
    CANN_NNAL_ENV_FILE="/usr/local/Ascend/nnal/atb/set_env.sh" && \
    echo "source ${CANN_TOOLKIT_ENV_FILE}" >> /etc/profile && \
    echo "source ${CANN_TOOLKIT_ENV_FILE}" >> ~/.bashrc && \
    echo "source ${CANN_NNAL_ENV_FILE}" >> /etc/profile && \
    echo "source ${CANN_NNAL_ENV_FILE}" >> ~/.bashrc

# 3. install torth & torth_npu
COPY ./torch*.whl /tmp/
RUN pip install --upgrade pip \
        && pip install $(ls -t /tmp/torch*.whl|grep -v "torch_npu"|head -n 1) \
        && pip install $(ls -t /tmp/torch_npu*.whl|head -n 1)

# 4. 适配MA
RUN useradd -m -d /home/ma-user -s /bin/bash -g 100 -u 1000 ma-user
USER ma-user
WORKDIR /home/ma-user
RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/ && pip config set global.timeout 6000
# RUN git config --global --unset http.proxy && git config --global --unset https.proxy
# 4. install Megatron-LM & MindSpeed & MindSpeed LLM
RUN cd ~ \
        && git clone https://github.com/NVIDIA/Megatron-LM.git \
        && git clone https://gitee.com/ascend/MindSpeed.git \
        && git clone https://gitee.com/ascend/MindSpeed-LLM.git

ARG MEGATRON_LM_TAG=core_r0.8.0
ARG MINDSPEED_TAG=2c085cc9
ARG MINDSPEED_LLM_TAG=2.1.0
RUN cd ~ && cd Megatron-LM && git checkout ${MEGATRON_LM_TAG} && cp -r megatron ../MindSpeed-LLM/ \
        && cd ../MindSpeed && git checkout ${MINDSPEED_TAG} && pip install -r requirements.txt && pip install -e . \
        && cd ../MindSpeed-LLM && git checkout ${MINDSPEED_LLM_TAG} && pip install -r requirements.txt

# 5. install transformer
ARG TRANSFORMERS_VERSION=
RUN if [ -n "$TRANSFORMERS_VERSION" ]; then echo "transformers is ${TRANSFORMERS_VERSION}" && pip install transformers==${TRANSFORMERS_VERSION}; else echo "transformers version is empty, skip"; fi

# 6. install apex
ARG APEX_TAG=master
RUN cd ~ && git clone https://gitee.com/ascend/apex.git \
        && export TORCH_DEVICE_BACKEND_AUTOLOAD=0 \
        && cd apex && git checkout ${APEX_TAG} && bash scripts/build.sh --python=$(python3 -V|awk '{print $2}'|cut -d. -f1,2) && cd apex/dist \
        && pip uninstall apex && pip install --upgrade $(ls -t apex*.whl|head -n 1) \
        && unset TORCH_DEVICE_BACKEND_AUTOLOAD

Please enter the base image of MindSpeed:

Please enter the tag or branch of Megatron-LM.git: (default: core_r0.8.0): 模型依赖Megatron-LM分支或者tag
Please enter the tag or branch of MindSpeed.git: (default: 2c085cc9): 模型依赖昇腾MindSpeed release分支或者tag
Please enter the tag or branch of MindSpeed-LLM.git: (default: 2.1.0): 模型依赖昇腾MindSpeed-LLM release分支或者tag
Please enter the version of transformers(nullable, skip when empty): 模型依赖transformers版本号，不设置跳过更新transformers
Please enter the tag or branch of apex.git: (default: master): 模型依赖昇腾apex release分支或者tag

tar -xvf obsutil_xxx.tar.gz
cp -f obsutil_xxx/obsutil /usr/local/bin/

obsutil config -i=<ak> -k=<sk> -e=<endpoint>
obsutil cp -r -f <local_weight_path> <obs_weight_path>

obsutil cp -r -f <local_data_path> <obs_data_path>

obsutil cp -r -f <local_script_path> <obs_script_path>

# set_env
source /usr/local/Ascend/driver/bin/setenv.bash
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/set_env.sh

export LD_LIBARY_PATH=/usr/local/Ascend/driver/lib64/driver:$LD_LIBARY_PATH
cd ~/MindSpeed-LLM/

# copy train origin data to dataset
cp $DATA_PATH/* ~/MindSpeed-LLM/dataset/
mkdir ./finetune_dataset
python ./preprocess_data.py \
    --input ./dataset \
    --tokenizer-name-or-path /home/mind/model/Qwen3-32B/ \
    --output-prefix ./finetune_dataset/alpaca \
    --handler-name AlpacaStyleInstructionHandler \
    --tokenizer-type PretrainedFromHF \
    --workers 16 \
    --log-interval 1000 \
    --prompt-type qwen

# set_env
source /usr/local/Ascend/driver/bin/setenv.bash
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/set_env.sh

export LD_LIBARY_PATH=/usr/local/Ascend/driver/lib64/driver:$LD_LIBARY_PATH
cd ~/MindSpeed-LLM/

mkdir -p $OUTPUT_PATH/qwen3_mcore
 
python convert_ckpt.py \
    --use-mcore-models \
    --model-type GPT \
    --load-model-type hf \
    --save-model-type mg \
    --target-tensor-parallel-size 4 \
    --target-pipeline-parallel-size 4 \
    --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
    --load-dir /home/mind/model/Qwen3-32B/ \
    --save-dir $OUTPUT_PATH/qwen3_mcore/ \
    --tokenizer-model /home/mind/model/Qwen3-32B/tokenizer.json \
    --model-type-hf qwen3 \
    --params-dtype bf16

# Change for multinode config
NPUS_PER_NODE="$MA_NUM_GPUS"           # 读取ModelArts下发的卡数
MASTER_ADDR="${VC_WORKER_HOSTS%%,*}"   # 设置MASTER节点HOST
MASTER_PORT=6000
NNODES="$MA_NUM_HOSTS"                 # 设置训练节点数
NODE_RANK="$VC_TASK_INDEX"             # 分配的节点索引
WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES)) # 计算总卡数

#!/bin/bash
export CUDA_DEVICE_MAX_CONNECTIONS=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
 
# set_env
source /usr/local/Ascend/driver/bin/setenv.bash
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/set_env.sh

export LD_LIBARY_PATH=/usr/local/Ascend/driver/lib64/driver:$LD_LIBARY_PATH
cd ~/MindSpeed-LLM/

mkdir -p $OUTPUT_PATH/ckpt

# Change for multinode config
# 读取ModelArts下发的卡数
NPUS_PER_NODE="$MA_NUM_GPUS"
# 设置MASTER节点HOST
MASTER_ADDR="${VC_WORKER_HOSTS%%,*}"
MASTER_PORT=6000
# 设置MA下发的节点数
NNODES="$MA_NUM_HOSTS"
# 设置MA分配的节点索引
NODE_RANK="$VC_TASK_INDEX"
# 计算总卡数
WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
 
# please fill these path configurations
CKPT_LOAD_DIR="$OUTPUT_PATH/qwen3_mcore/"
CKPT_SAVE_DIR="$OUTPUT_PATH/ckpt/"
DATA_PATH="./finetune_dataset/alpaca "
TOKENIZER_PATH="/home/mind/model/Qwen3-32B"
 
# TP\PP\MBS\GBS\SEQ_LENGTH\TRAIN_ITERS 按照需求设置
TP=4
PP=4
MBS=2
GBS=16
SEQ_LENGTH=4096
TRAIN_ITERS=2000


DISTRIBUTED_ARGS="
    --nproc_per_node $NPUS_PER_NODE \
    --nnodes $NNODES \
    --node_rank $NODE_RANK \
    --master_addr $MASTER_ADDR \
    --master_port $MASTER_PORT
"

OPTIMIZE_ARGS="
    --use-flash-attn \
    --use-fused-rotary-pos-emb \
    --use-rotary-position-embeddings \
    --use-fused-swiglu \
    --use-fused-rmsnorm \
    --no-masked-softmax-fusion \
    --use-distributed-optimizer
"

TRAIN_ARGS="
    --micro-batch-size ${MBS} \
    --global-batch-size ${GBS} \
    --lr 1.25e-6 \
    --lr-decay-style cosine \
    --min-lr 1.25e-7 \
    --weight-decay 1e-1 \
    --lr-warmup-fraction 0.01 \
    --attention-dropout 0.0 \
    --init-method-std 0.01 \
    --hidden-dropout 0.0 \
    --clip-grad 1.0 \
    --adam-beta1 0.9 \
    --adam-beta2 0.95 \
    --initial-loss-scale 4096 \
    --seed 42 \
    --bf16 \
    --train-iters ${TRAIN_ITERS} \
    --seq-length ${SEQ_LENGTH} \
    --no-shared-storage
"

MODEL_PARALLEL_ARGS="
    --tensor-model-parallel-size ${TP} \
    --pipeline-model-parallel-size ${PP}
"

GPT_ARGS="
    --use-mcore-models \
    --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
    --kv-channels 128 \
    --qk-layernorm \
    --tokenizer-name-or-path ${TOKENIZER_PATH} \
    --max-position-embeddings ${SEQ_LENGTH} \
    --num-layers 64 \
    --hidden-size 5120 \
    --ffn-hidden-size 25600 \
    --num-attention-heads 64 \
    --tokenizer-type PretrainedFromHF \
    --make-vocab-size-divisible-by 1 \
    --padded-vocab-size 151936 \
    --rotary-base 1000000 \
    --untie-embeddings-and-output-weights \
    --disable-bias-linear \
    --position-embedding-type rope \
    --normalization RMSNorm \
    --swiglu \
    --attention-softmax-in-fp32 \
    --no-gradient-accumulation-fusion \
    --group-query-attention \
    --num-query-groups 8
"

DATA_ARGS="
    --data-path $DATA_PATH \
    --split 100,0,0
"

OUTPUT_ARGS="
    --load ${CKPT_LOAD_DIR} \
	--save ${CKPT_SAVE_DIR} \
    --log-interval 1 \
    --save-interval ${TRAIN_ITERS} \
    --eval-interval ${TRAIN_ITERS} \
    --eval-iters 0 \
    --no-load-optim \
    --no-load-rng
"

TUNE_ARGS="
    --finetune \
    --stage sft \
    --is-instruction-dataset \
    --tokenizer-not-use-fast \
    --prompt-type qwen \
    --variable-seq-lengths
"

torchrun $DISTRIBUTED_ARGS posttrain_gpt.py \
    $GPT_ARGS \
    $DATA_ARGS \
    $OUTPUT_ARGS \
    $OPTIMIZE_ARGS \
    $TRAIN_ARGS \
    $TUNE_ARGS \
    $MODEL_PARALLEL_ARGS \
    --distributed-backend nccl \
    | tee logs/tune_qwen3_32b_full_ptd.log

#!/bin/bash
export CUDA_DEVICE_MAX_CONNECTIONS=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
 
# set_env
source /usr/local/Ascend/driver/bin/setenv.bash
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/set_env.sh

export LD_LIBARY_PATH=/usr/local/Ascend/driver/lib64/driver:$LD_LIBARY_PATH

cd ~/MindSpeed-LLM/

mkdir -p $OUTPUT_PATH/ckpt

export HCCL_CONNECT_TIMEOUT=1800

# Change for multinode config
# 读取ModelArts下发的卡数
NPUS_PER_NODE="$MA_NUM_GPUS"
# 设置MA节点HOST
MASTER_ADDR="${VC_WORKER_HOSTS%%,*}"
MASTER_PORT=6014
# 设置MA下发的节点数
NNODES="$MA_NUM_HOSTS"
# 设置MA分配的节点索引
NODE_RANK="$VC_TASK_INDEX"
# 计算总卡数
WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))

# please fill these path configurations
CKPT_LOAD_DIR="$OUTPUT_PATH/qwen3_mcore/"
CKPT_SAVE_DIR="$OUTPUT_PATH/ckpt/"
DATA_PATH="./finetune_dataset/alpaca "
TOKENIZER_PATH="/home/mind/model/Qwen3-32B"

TP=4
PP=4
SEQ_LENGTH=4096
TRAIN_ITERS=2000


DISTRIBUTED_ARGS="
    --nproc_per_node $NPUS_PER_NODE \
    --nnodes $NNODES \
    --node_rank $NODE_RANK \
    --master_addr $MASTER_ADDR \
    --master_port $MASTER_PORT
"

OPTIMIZE_ARGS="
    --use-flash-attn \
    --use-fused-rotary-pos-emb \
    --use-rotary-position-embeddings \
    --use-fused-swiglu \
    --use-fused-rmsnorm \
    --no-masked-softmax-fusion \
    --use-distributed-optimizer
"

TRAIN_ARGS="
    --micro-batch-size 4 \
    --global-batch-size 16 \
    --lr 1.25e-5 \
    --lr-decay-style cosine \
    --min-lr 1.25e-7 \
    --weight-decay 1e-1 \
    --lr-warmup-fraction 0.01 \
    --attention-dropout 0.0 \
    --init-method-std 0.01 \
    --hidden-dropout 0.0 \
    --clip-grad 1.0 \
    --adam-beta1 0.9 \
    --adam-beta2 0.95 \
    --initial-loss-scale 4096 \
    --seed 42 \
    --bf16 \
    --train-iters ${TRAIN_ITERS} \
    --seq-length ${SEQ_LENGTH} \
    --no-shared-storage
"

MODEL_PARALLEL_ARGS="
    --tensor-model-parallel-size ${TP} \
    --pipeline-model-parallel-size ${PP}
"

GPT_ARGS="
    --use-mcore-models \
    --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
    --kv-channels 128 \
    --qk-layernorm \
    --tokenizer-name-or-path ${TOKENIZER_PATH} \
    --max-position-embeddings ${SEQ_LENGTH} \
    --num-layers 64 \
    --hidden-size 5120 \
    --ffn-hidden-size 25600 \
    --num-attention-heads 64 \
    --tokenizer-type PretrainedFromHF \
    --make-vocab-size-divisible-by 1 \
    --padded-vocab-size 151936 \
    --rotary-base 1000000 \
    --untie-embeddings-and-output-weights \
    --disable-bias-linear \
    --position-embedding-type rope \
    --normalization RMSNorm \
    --swiglu \
    --attention-softmax-in-fp32 \
    --no-gradient-accumulation-fusion \
    --group-query-attention \
    --num-query-groups 8
"

DATA_ARGS="
    --data-path $DATA_PATH \
    --split 100,0,0
"

OUTPUT_ARGS="
    --load ${CKPT_LOAD_DIR} \
	--save ${CKPT_SAVE_DIR} \
    --log-interval 1 \
    --save-interval ${TRAIN_ITERS} \
    --eval-interval ${TRAIN_ITERS} \
    --eval-iters 0 \
    --no-load-optim \
    --no-load-rng
"

TUNE_ARGS="
    --finetune \
    --stage sft \
    --is-instruction-dataset \
    --tokenizer-not-use-fast \
    --prompt-type qwen \
    --variable-seq-lengths \
    --lora-r 16 \
    --lora-alpha 32 \
    --lora-fusion \
    --lora-target-modules linear_qkv linear_proj linear_fc1 linear_fc2
"

torchrun $DISTRIBUTED_ARGS posttrain_gpt.py \
    $GPT_ARGS \
    $DATA_ARGS \
    $OUTPUT_ARGS \
    $OPTIMIZE_ARGS \
    $TRAIN_ARGS \
    $TUNE_ARGS \
    $MODEL_PARALLEL_ARGS \
    --distributed-backend nccl \
    | tee logs/tune_qwen3_32b_lora_ptd.log

# set_env
source /usr/local/Ascend/driver/bin/setenv.bash
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/set_env.sh

export LD_LIBARY_PATH=/usr/local/Ascend/driver/lib64/driver:$LD_LIBARY_PATH

# master节点进行最后的权重转换
if [[ "$VC_TASK_INDEX" == 0 ]]; then
    # 汇总权重到OBS挂载路径，需要自行汇总
    echo "This is master, doing mcore to hf"
    # 汇总所有节点权重，OBS地址可以从MA_OUTPUTS环境变量中获取romote，主要需要登录，todo：可以使用moxing解决，目前已跟产品线沟通发布外部商发版本
    obsutil cp -r -f obs://qwen3/llmd/ckpt/ $OUTPUT_PATH/
    mkdir -p $OUTPUT_PATH/qwen3_hf
    # 这个转换需要目录下有原始权重，cp到output下
    cp -rf /home/mind/model/Qwen3-32B/* $OUTPUT_PATH/qwen3_hf

    python convert_ckpt.py \
        --use-mcore-models \
        --model-type GPT \
        --load-model-type mg \
        --save-model-type hf \
        --target-tensor-parallel-size 1 \
        --target-pipeline-parallel-size 1 \
        --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
        --load-dir $OUTPUT_PATH/ckpt/ \
        --save-dir $OUTPUT_PATH/qwen3_hf/ \
        --model-type-hf qwen3 \
        --params-dtype bf16
    else
        echo "This is slave, ignore mcore to hf"
fi

# 修改 ascend-toolkit 路径
source /usr/local/Ascend/driver/bin/setenv.bash
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/set_env.sh

export LD_LIBARY_PATH=/usr/local/Ascend/driver/lib64/driver:$LD_LIBARY_PATH

if [[ "$VC_TASK_INDEX" == 0 ]]; then
    # 汇总权重到OBS挂载路径，需要自行汇总
    echo "This is master, doing mcore to hf"
    # 汇总所有节点权重，OBS地址可以从MA_OUTPUTS环境变量中获取romote，主要需要登录，todo：可以使用moxing解决，目前已跟产品线沟通发布外部商发版本
    obsutil cp -r -f obs://qwen3/llmd/ckpt/ $OUTPUT_PATH/
    mkdir -p $OUTPUT_PATH/qwen3_hf
    # 这个转换需要目录下有原始权重
    cp -rf /home/mind/model/Qwen3-32B/* $OUTPUT_PATH/qwen3_hf

    python convert_ckpt.py \
        --use-mcore-models \
        --model-type GPT \
        --load-model-type mg \
        --save-model-type hf \
        --target-tensor-parallel-size 1 \
        --target-pipeline-parallel-size 1 \
        --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
        --lora-r 16 \
        --lora-alpha 32 \
        --lora-target-modules linear_qkv linear_proj linear_fc1 linear_fc2 \
        --load-dir $OUTPUT_PATH/qwen3_mcore/ \
        --lora-load $OUTPUT_PATH/ckpt/ \
        --save-dir $OUTPUT_PATH/qwen3_hf/ \
        --model-type-hf qwen3 \
fi

bash $SCRIPT_PATH/data_convert_qwen3_32b_instruction.sh \
&& bash $SCRIPT_PATH/ckpt_convert_qwen3_32b_hf2mcore.sh \
&& bash $SCRIPT_PATH/tune_qwen3_32b_4K_full_ptd.sh \
&& sleep 1000 && bash $SCRIPT_PATH/ckpt_convert_qwen3_mcore2hf.sh

# set_env
source /usr/local/Ascend/driver/bin/setenv.bash
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/set_env.sh
export LD_LIBARY_PATH=/usr/local/Ascend/driver/lib64/driver:$LD_LIBARY_PATH