ERNIE-4.5-VL-28B-A3B-Thinking Ascend NPU 适配

# 方式一：使用 modelscope CLI
pip install modelscope
modelscope download --model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking --local_dir /model/ERNIE-4.5-VL-28B-A3B-Thinking

# 方式二：使用 git clone（需要 git-lfs）
git lfs install
git clone https://www.modelscope.cn/PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking.git /model/ERNIE-4.5-VL-28B-A3B-Thinking

/model/ERNIE-4.5-VL-28B-A3B-Thinking/
├── config.json
├── modeling_ernie4_5_vl.py
├── processing_ernie4_5_vl.py
├── tokenizer_config.json
├── vocab.txt
├── pytorch_model-00001-of-000XX.bin
├── pytorch_model-00002-of-000XX.bin
└── ...

# 创建容器（推荐使用 --init 防止僵尸进程）
docker run -d --init --name ernie-vllm \
  --privileged \
  --shm-size=16g \
  --device=/dev/davinci_manager \
  --device=/dev/devmm_svm \
  --device=/dev/hisi_hdc \
  $(for i in $(seq 0 $((NPU_COUNT-1))); do echo "--device=/dev/davinci$i "; done) \
  -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/:ro \
  -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info:ro \
  -v <model_path>:/model \
  -v <data_path>:/data \
  -v <code_path>:/code \
  -p <ssh_port>:22 \
  -p 8010:8010 \
  quay.io/ascend/vllm-ascend:v0.18.0rc1-a3 \
  sleep infinity

# 配置 SSH（如需远程连接）
docker exec ernie-vllm bash -c "
apt-get update && apt-get install -y openssh-server
mkdir -p /run/sshd
ssh-keygen -A
sed -i 's/#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config
sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config
echo 'root:<your_password>' | chpasswd
/usr/sbin/sshd
"

# 设置可见 NPU 设备（根据实际空闲设备调整）
export ASCEND_RT_VISIBLE_DEVICES=0,1

# CANN 环境变量
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh

export LD_LIBRARY_PATH=/usr/local/Ascend/cann-8.5.1/lib64:$LD_LIBRARY_PATH
export ASCEND_OPP_PATH=/usr/local/Ascend/cann-8.5.1/opp
export ASCEND_TOOLKIT_HOME=/usr/local/Ascend/cann-8.5.1
export ASCEND_HOME_PATH=/usr/local/Ascend/cann-8.5.1

# 启动推理服务
vllm serve /model/ERNIE-4.5-VL-28B-A3B-Thinking \
  --served-model-name ernie-28B \
  --tensor-parallel-size 2 \
  --trust-remote-code \
  --max-model-len 10240 \
  --host 0.0.0.0 \
  --port 8010

pip install datasets pillow openai

# FFmpeg（系统级依赖）
apt-get update && apt-get install -y ffmpeg

# decord（视频解码库）
pip install decord

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8010/v1",
    api_key="not-needed"
)

# 纯文本推理
response = client.chat.completions.create(
    model="ernie-28B",
    messages=[{"role": "user", "content": "你好，请介绍一下你自己"}],
    max_tokens=1024,
    temperature=0.0
)
print(response.choices[0].message.content)

# 多模态推理（带图像）
import base64
from PIL import Image
import io

def pil_to_base64(img):
    buf = io.BytesIO()
    img.save(buf, format="PNG")
    return base64.b64encode(buf.getvalue()).decode()

# 加载图像并推理
img = Image.open("your_image.png")
response = client.chat.completions.create(
    model="ernie-28B",
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{pil_to_base64(img)}"}},
                {"type": "text", "text": "请描述这张图片的内容"}
            ]
        }
    ],
    max_tokens=1024
)
print(response.choices[0].message.content)

# 训练相关依赖
pip install peft accelerate datasets

# 注意：torch 和 torch_npu 通常已在 Ascend 镜像中预装
# transformers 版本需兼容模型（建议 4.57.6+）

# 设置环境变量
source /usr/local/Ascend/ascend-toolkit/set_env.sh
export ASCEND_RT_VISIBLE_DEVICES=0

# 使用合成数据训练（快速验证）
python -u lora_train.py --epochs 3 --max_length 256 --lr 1e-4 --use_fallback

# 使用 COIG-CQIA 数据集训练
python -u lora_train.py --epochs 3 --max_length 32 --lr 1e-4 --max_samples 500

# 使用本地数据文件训练
python -u lora_train.py --epochs 3 --max_length 32 --lr 1e-4 --data_path /code/train_data.jsonl

trainable params: 1,133,568 || all params: 29,400,427,904 || trainable%: 0.0039
[Epoch 1/3] avg_loss=2.6406, mem=58.83GB
[Epoch 2/3] avg_loss=2.6406, mem=58.83GB
[Epoch 3/3] avg_loss=2.6406, mem=58.83GB
Training complete! 60 steps in 121.8s

trainable params: 1,133,568 || all params: 29,400,427,904 || trainable%: 0.0039
[Epoch 1/3] avg_loss=4.5056, mem=58.84GB
[Epoch 2/3] avg_loss=4.5056, mem=58.84GB
[Epoch 3/3] avg_loss=4.5056, mem=58.84GB
Training complete! 1500 steps in 3533.8s

# 环境变量（必须）
ASCEND_RT_VISIBLE_DEVICES=0,1

# vLLM 参数
--tensor-parallel-size 2    # 2卡并行
--max-model-len 10240       # 最大上下文长度
--trust-remote-code         # 信任模型代码

# 推理参数
max_tokens=8192             # 输出长度（Thinking 模型需较长）
temperature=0.0             # 确定性输出