GLM-4.7-Flash 在 Ascend NPU 上的 vLLM 适配说明

cd /path/to/glm-4.7-flash
PATCH_DIR=$(pwd)

cd /vllm-workspace/vllm
git apply --check "${PATCH_DIR}/vllm-v0.17.0rc1-glm47flash.patch"
git apply "${PATCH_DIR}/vllm-v0.17.0rc1-glm47flash.patch"

cd /vllm-workspace/vllm-ascend
git apply --check "${PATCH_DIR}/vllm-ascend-v0.17.0rc1-glm47flash.patch"
git apply "${PATCH_DIR}/vllm-ascend-v0.17.0rc1-glm47flash.patch"

ls /vllm-workspace/vllm/vllm/model_executor/models/glm4_moe_lite*.py
ls /vllm-workspace/vllm/vllm/transformers_utils/configs/glm4_moe_lite.py

HCCL_OP_EXPANSION_MODE=AIV vllm serve /models/GLM-4.7-Flash \
  --served-model-name GLM-4.7-Flash \
  --tensor-parallel-size 4 \
  --max-model-len 32768 \
  --port 8000

HCCL_OP_EXPANSION_MODE=AIV vllm serve /models/GLM-4.7-Flash \
  --served-model-name GLM-4.7-Flash \
  --tensor-parallel-size 4 \
  --max-model-len 32768 \
  --enable-expert-parallel \
  --speculative-config '{"method":"mtp","num_speculative_tokens":1}' \
  --port 8013

qk_nope_head_dim = 192   # Query/Key 非位置编码维度
qk_rope_head_dim = 64    # Query/Key 的 RoPE 维度
v_head_dim = 256         # Value 维度
kv_lora_rank = 512       # KV 压缩秩
q_lora_rank = 768        # Query 压缩秩

# 在 AscendMLAImpl 内部，self.num_heads 表示单卡本地头数
local_num_heads = self.num_heads       # TP=4 时为 5
need_head_padding = not _is_power_of_2(local_num_heads)
padded_num_heads = _next_power_of_2(local_num_heads)  # 5 -> 8

# 对 Q 张量进行填充
if need_head_padding:
    q_nope = torch.nn.functional.pad(q_nope, (0, 0, 0, pad_heads))
    q_pe = torch.nn.functional.pad(q_pe, (0, 0, 0, pad_heads))

# 计算后裁剪回原始头数
if need_head_padding:
    attn_output = attn_output[:local_num_heads]

def _forward_prefill_fallback(self, q_nope, q_pe, k_nope, k_pe, value, ...):
    # 合并 Q/K 的 nope 和 pe，满足 query_dim >= value_dim 约束
    query = torch.cat([q_nope, q_pe], dim=-1)  # [T, N, 256]
    key = torch.cat([k_nope, k_pe], dim=-1)    # [T, N, 256]

    # 使用 BNSD 格式 + sparse_mode=3，逐序列计算
    for seq_len in query_lens_list:
        causal_mask = self._get_prefill_fallback_causal_mask(seq_len, query.device)
        q_bnsd = query[start:end].transpose(0, 1).unsqueeze(0)  # [1, N, S, D]
        attn_out, lse = torch_npu.npu_fused_infer_attention_score(
            query=q_bnsd, key=k_bnsd, value=v_bnsd,
            input_layout="BNSD", sparse_mode=3, atten_mask=causal_mask,
            softmax_lse_flag=True,
        )

    # chunked-prefix: 从 paged KV cache 取回历史上下文，
    # 用 float32 logsumexp 风格合并，避免长序列重复问题
    if chunked_context is not None:
        for chunk_idx, toks in enumerate(chunked_context.seq_tot):
            # npu_paged_cache_load -> kv_b_proj -> attention -> merge
            merged_output = base_output * exp(base_lse - merged_lse) \
                          + update_output * exp(update_lse - merged_lse)

BASE_URL=http://127.0.0.1:8000  # 若使用 EP + MTP 快路径则改为 http://127.0.0.1:8013

curl -s "${BASE_URL}/v1/chat/completions" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "GLM-4.7-Flash",
    "messages": [
      {"role": "user", "content": "What is the capital of France?"}
    ],
    "temperature": 0,
    "max_tokens": 32
  }' | python -m json.tool

{
  "model": "GLM-4.7-Flash",
  "messages": [
    {"role": "user", "content": "Continue this sentence in one short line: San Francisco is a"}
  ],
  "temperature": 0,
  "max_tokens": 32
}