MiniCPM-V-4.6-Thinking 在 vLLM-Ascend 上的部署适配

from transformers.configuration_utils import PretrainedConfig
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5TextConfig


class MiniCPMV4_6VisionConfig(PretrainedConfig):
    model_type = "minicpmv4_6_vision"
    def __init__(self, hidden_size=1152, num_hidden_layers=27, num_attention_heads=16,
                 intermediate_size=4304, hidden_act="gelu_pytorch_tanh", patch_size=14,
                 image_size=980, layer_norm_eps=1e-6, num_channels=3, attention_dropout=0.0,
                 **kwargs):
        super().__init__(**kwargs)
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.patch_size = patch_size
        self.image_size = image_size
        self.layer_norm_eps = layer_norm_eps
        self.num_channels = num_channels
        self.attention_dropout = attention_dropout


class MiniCPMV4_6Config(PretrainedConfig):
    model_type = "minicpmv4_6"
    def __init__(self, text_config=None, vision_config=None, image_size=1120,
                 image_token_id=248056, video_token_id=248057, vision_start_token_id=248053,
                 vision_end_token_id=248054, insert_layer_id=6, query_num=64,
                 drop_vision_last_layer=False, tie_word_embeddings=False, use_cache=True, **kwargs):
        self.image_size = image_size
        self.image_token_id = image_token_id
        self.video_token_id = video_token_id
        self.vision_start_token_id = vision_start_token_id
        self.vision_end_token_id = vision_end_token_id
        self.insert_layer_id = insert_layer_id
        self.query_num = query_num
        self.drop_vision_last_layer = drop_vision_last_layer
        self.tie_word_embeddings = tie_word_embeddings
        self.use_cache = use_cache
        if isinstance(vision_config, dict):
            self.vision_config = MiniCPMV4_6VisionConfig(**vision_config)
        elif vision_config is None:
            self.vision_config = MiniCPMV4_6VisionConfig()
        else:
            self.vision_config = vision_config
        if isinstance(text_config, dict):
            self.text_config = Qwen3_5TextConfig(**text_config)
        elif text_config is None:
            self.text_config = Qwen3_5TextConfig()
        else:
            self.text_config = text_config
        super().__init__(**kwargs)

_CONFIG_REGISTRY["minicpmv4_6"] = "MiniCPMV4_6Config"

_CLASS_TO_MODULE["MiniCPMV4_6Config"] = "vllm.transformers_utils.configs.minicpmv4_6"
_CLASS_TO_MODULE["MiniCPMV4_6VisionConfig"] = "vllm.transformers_utils.configs.minicpmv4_6"
__all__.extend(["MiniCPMV4_6Config", "MiniCPMV4_6VisionConfig"])

def get_version_by_config(config) -> tuple[int, ...]:
    version_float = getattr(config, "version", None)
    if version_float is None:
        if getattr(config, "model_type", None) == "minicpmv4_6":
            return (4, 6)
        if config.hidden_size == 2304 and config.query_num == 64:
            return (2, 0)
        return (2, 5)
    version_str = str(version_float)
    return tuple(int(x) for x in version_str.split("."))

def __new__(cls, *, vllm_config, prefix=""):
    config = vllm_config.model_config.hf_config
    if getattr(config, "model_type", None) == "minicpmv4_6":
        version = (4, 6)
    elif not hasattr(config, "version"):
        if config.hidden_size == 2304 and config.query_num == 64:
            version = (2, 0)
        else:
            version = (2, 5)
    else:
        version = str(config.version).split(".")
        version = tuple([int(x) for x in version])
    instance_cls = _SUPPORT_VERSION.get(version)
    ...

class MiniCPMV4_6ForConditionalGeneration(MiniCPMVBaseModel, SupportsLoRA):
    def __init__(self, *, vllm_config, prefix=""):
        super().__init__(vllm_config=vllm_config, prefix=prefix)
        assert self.version == (4, 6)

    def init_llm(self, vllm_config, prefix=""):
        return Qwen3_5ForCausalLM(vllm_config=vllm_config, prefix=prefix)

    def init_vision_module(self, config, quant_config=None, prefix=""):
        model = Idefics2VisionTransformer(
            config.vision_config,
            quant_config=quant_config,
            apply_encoder_attention_mask=True,
            prefix=prefix,
        )
        if self.config.drop_vision_last_layer:
            model.encoder.layers = model.encoder.layers[:-1]
        return model

    def init_resampler(self, embed_dim, vision_dim, quant_config=None, prefix=""):
        with set_default_torch_dtype(torch.float16):
            resampler = Resampler4_5(
                num_queries=self.config.query_num,
                embed_dim=embed_dim,
                num_heads=embed_dim // 128,
                kv_dim=vision_dim,
                quant_config=quant_config,
                prefix=f"{prefix}.resampler",
            )
        return resampler

    def get_vision_hidden_states(self, data):
        raise NotImplementedError("Need image processor")

    def load_weights(self, weights):
        from vllm.model_executor.models.auto_weights import AutoWeightsLoader
        loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"])
        return loader.load_weights(weights)


_SUPPORT_VERSION[(4, 6)] = MiniCPMV4_6ForConditionalGeneration

export ASCEND_RT_VISIBLE_DEVICES=0
export VLLM_USE_MODELSCOPE=true
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export TASK_QUEUE_ENABLE=1

vllm serve /mnt/weight/MiniCPM-V-4.6-Thinking \
  --host 0.0.0.0 \
  --port 8000 \
  --tensor-parallel-size 1 \
  --seed 1024 \
  --served-model-name minicpm-v-4.6-thinking \
  --max-model-len 8192 \
  --max-num-seqs 16 \
  --trust-remote-code \
  --gpu-memory-utilization 0.90 \
  --no-enable-prefix-caching \
  --dtype bfloat16 \
  --enforce-eager

curl -sf http://127.0.0.1:8000/v1/models | python3 -m json.tool

curl -sf http://127.0.0.1:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "minicpm-v-4.6-thinking",
    "messages": [
      {"role": "user", "content": "请计算 23 × 47 = ?，写出详细步骤。"}
    ],
    "temperature": 0.6,
    "max_tokens": 512,
    "top_p": 0.9
  }'

curl -sf http://127.0.0.1:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "minicpm-v-4.6-thinking",
    "messages": [{
      "role": "user",
      "content": [
        {"type": "image_url", "image_url": {"url": "https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png"}},
        {"type": "text", "text": "请观察图片，分析这个物理现象的原因。"}
      ]
    }],
    "temperature": 0.6,
    "max_tokens": 1024,
    "top_p": 0.9
  }'

每层总误差 = Linear(1.5e-5) + LN(1e-5) + Attn(5e-4) + Act(2e-5)
           ≈ 5.45e-4 per layer (linear_attn层)
           ≈ 5.45e-4 per layer (full_attn层)

24层累积 → 24 × 5.45e-4 ≈ 0.013
最终 logits 层 → Softmax归一化后 logit 差异 ≈ 0.01 - 0.03

# Step 1: 在 GPU 上运行参考
python -c "
from vllm import LLM, SamplingParams
model = LLM('OpenBMB/MiniCPM-V-4.6-Thinking', trust_remote_code=True, dtype='bfloat16')
prompts = ['What is 2+3?', 'Calculate 15 × 27']
outputs = model.generate(prompts)
for o in outputs:
    print(f'GPU: {o.outputs[0].text}')
"

# Step 2: 在 NPU 上运行相同输入
python -c "
from vllm import LLM, SamplingParams
model = LLM('OpenBMB/MiniCPM-V-4.6-Thinking', trust_remote_code=True, dtype='bfloat16')
prompts = ['What is 2+3?', 'Calculate 15 × 27']
outputs = model.generate(prompts)
for o in outputs:
    print(f'NPU: {o.outputs[0].text}')
"

# Step 3: 对比 logits（使用相同的随机种子）
python -c "
import torch
# 在 GPU 上加载模型
model_gpu = ...  # CUDA 加载
logits_gpu = model_gpu.forward(input_ids)

# 在 NPU 上加载模型
model_npu = ...  # NPU 加载
logits_npu = model_npu.forward(input_ids)

# 计算差异
diff = (logits_gpu - logits_npu).abs()
print(f'Max logit diff: {diff.max().item():.6f}')
print(f'Mean logit diff: {diff.mean().item():.6f}')
print(f'Relative error: {diff.mean().item() / logits_gpu.abs().mean().item() * 100:.3f}%')
"

def get_mm_mapping(self) -> MultiModelKeys:
    return MultiModelKeys.from_files(
        llm={"model.language_model.": "llm."},
        vpm={"model.vision_tower.": "vpm."},
        resampler={"model.resampler.": "resampler."},
    )

model.language_model.embed_tokens.weight         [248094, 1024]   BF16
model.language_model.layers.0.input_layernorm    [1024]           BF16
model.language_model.layers.0.linear_attn.*      GatedDeltaNet     BF16
  ├── A_log                                      [16]             BF16
  ├── conv1d.weight                              [6144, 1, 4]     BF16
  ├── dt_bias                                    [16]             BF16
  ├── in_proj_a.weight                           [16, 1024]       BF16
  ├── in_proj_b.weight                           [16, 1024]       BF16
  ├── in_proj_qkv.weight                         [6144, 1024]     BF16
  ├── in_proj_z.weight                           [2048, 1024]     BF16
  ├── norm.weight                                [128]            BF16
  └── out_proj.weight                            [1024, 2048]     BF16
model.language_model.layers.0.mlp.*              MLP (SiLU)
  ├── gate_proj.weight                           [3584, 1024]     BF16
  ├── up_proj.weight                             [3584, 1024]     BF16
  └── down_proj.weight                           [1024, 3584]     BF16
model.language_model.layers.0.post_attention_layernorm [1024]     BF16
...
model.language_model.layers.3.self_attn.*        Full Attention
  ├── q_proj.weight                              [2048, 1024]     BF16
  ├── k_proj.weight                              [512, 1024]      BF16
  ├── v_proj.weight                              [512, 1024]      BF16
  ├── o_proj.weight                              [1024, 2048]     BF16
  ├── q_norm.weight                              [256]            BF16
  └── k_norm.weight                              [256]            BF16
...
model.language_model.norm.weight                 [1024]           BF16
model.language_model.lm_head.weight              [248094, 1024]   BF16
---
model.vision_tower.vision_model.*                ViT encoder       BF16
model.vision_tower.vit_merger.*                  Visual Merger     BF16
model.resampler.*                                Resampler         BF16