GLM-5 SGLang 910B 硬件成功部署指南和经验教训

npu-smi info  # 查看 NPU 状态和显存

python3 -c "import torch, torch_npu, sglang, transformers; \
print(f'torch: {torch.__version__}'); \
print(f'torch_npu: {torch_npu.__version__}'); \
print(f'sglang: {sglang.__version__}'); \
print(f'transformers: {transformers.__version__}'); \
import triton.backends.ascend; print('triton-ascend: OK')"

cosdt-modelscope/sglang:cann8.5.0-910b-glm5

ValueError: The checkpoint you are trying to load has model type `glm_moe_dsa` 
but Transformers does not recognize this architecture.

# 升级 transformers
pip install --upgrade transformers==5.2.0

# 确认版本
pip show transformers | grep Version
# 输出: Version: 5.2.0

sglang 0.1.dev9598 requires transformers==4.57.1, but you have transformers 5.2.0

pip list | grep triton

triton-ascend    3.2.0

triton           3.6.0    # ← 这个会导致冲突！
triton-ascend    3.2.0

# 1. 卸载冲突的 triton 包
pip uninstall -y triton

# 2. 重新安装 triton-ascend 以修复符号链接
pip install --force-reinstall --no-deps triton-ascend==3.2.0

# 3. 验证 NPU 后端可用
python3 -c "import triton; import triton.backends.ascend; print('✅ Triton-Ascend NPU backend OK')"

$ python3 -c "import triton.backends.ascend"
# 应该没有任何输出（无错误）

#!/bin/bash
# GLM-5 SGLang deployment with DeepEP-Ascend optimized settings
# Hardware: Atlas 800 A2 (910B2C)
# Date: 2026-03-02

# ============================================================
# Model Configuration
# ============================================================
export MODEL_PATH=""

# ============================================================
# System Optimization
# ============================================================
echo "Configuring CPU for high performance..."
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor 2>/dev/null || true
sysctl -w vm.swappiness=0 2>/dev/null || true
sysctl -w kernel.numa_balancing=0 2>/dev/null || true
sysctl -w kernel.sched_migration_cost_ns=50000 2>/dev/null || true

# ============================================================
# Network Proxy (Disable)
# ============================================================
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING

# ============================================================
# CANN Environment
# ============================================================
echo "Loading CANN environment..."
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh

# ============================================================
# HCCL Configuration (Communication)
# ============================================================
export HCCL_INTRA_ROCE_ENABLE=1
export HCCL_BUFFSIZE=1024
export HCCL_DETERMINISTIC="true"
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo

# ============================================================
# NPU Memory and Threading
# ============================================================
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export OMP_NUM_THREADS=1
export TASK_QUEUE_ENABLE=1

# ============================================================
# SGLang Configuration
# ============================================================
export SGLANG_BACKEND=rpc
export SGLANG_DEVICE=npu
export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export STREAMS_PER_DEVICE=32

# ============================================================
# CUDA Graph Settings (for reference, disabled in launch)
# ============================================================
export VLLM_CUDAGRAPH_MODE="FULL_DECODE_ONLY"
export VLLM_CUDAGRAPH_CAPTURE_SIZES="[1,4,8,16,32,48,64]"

# ============================================================
# Launch Server
# ============================================================
echo "============================================"
echo "Starting SGLang server for GLM-5-w4a8"
echo "============================================"
echo "Hardware: Atlas 800 A2 (910B2C)"
echo "NPU Count: 16"
echo "DeepEP Mode: low_latency"
echo "Max Tokens/Batch: 8192 (A2 hardware limit)"
echo "============================================"

python3 -m sglang.launch_server \
  --model-path $MODEL_PATH \
  --attention-backend ascend \
  --device npu \
  --tp-size 16 \
  --nnodes 1 \
  --node-rank 0 \
  --chunked-prefill-size 8192 \
  --max-prefill-tokens 8192 \
  --trust-remote-code \
  --host 0.0.0.0 \
  --mem-fraction-static 0.7 \
  --port 8000 \
  --served-model-name glm-5-w4a8 \
  --disable-cuda-graph \
  --quantization modelslim \
  --moe-a2a-backend deepep \
  --deepep-mode low_latency \
  --skip-server-warmup

# 添加执行权限
chmod +x deploy_glm5_sglang_deepep_optimized.sh

# 后台启动（推荐）
bash deploy_glm5_sglang_deepep_optimized.sh > glm5_sglang.log 2>&1 &

# 查看日志
tail -f glm5_sglang.log

# 检查服务状态
curl http://localhost:8000/v1/models

# 环境变量保持不变
export VLLM_CUDAGRAPH_MODE="FULL_DECODE_ONLY"
export VLLM_CUDAGRAPH_CAPTURE_SIZES="[1,4,8,16,32,48,64]"

# 启动参数：移除 --disable-cuda-graph
python3 -m sglang.launch_server \
  --model-path $MODEL_PATH \
  ... \
  # --disable-cuda-graph  ← 删除此行，启用 CUDA Graph
  --quantization modelslim \
  ...

[2026-03-02 14:57:58] Capture npu graph begin. This can take up to several minutes.
[2026-03-02 14:57:58] Capture cuda graph bs [16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256]

Capturing batches (bs=256 avail_mem=15.08 GB):   6%|▋  | 1/16 [00:08<02:10,  8.68s/it]
Capturing batches (bs=240 avail_mem=10.03 GB):  12%|█▎ | 2/16 [00:13<01:29,  6.41s/it]
...
Capturing batches (bs=16 avail_mem=9.17 GB):   100%|██████████| 16/16 [01:11<00:00,  4.46s/it]

[2026-03-02 14:59:10] Capture npu graph end. Time elapsed: 72.08 s. mem usage=6.11 GB. avail mem=9.20 GB.
[2026-03-02 14:59:12] max_total_num_tokens=151168, chunked_prefill_size=8192
[2026-03-02 14:59:13] The server is fired up and ready to roll!

KV Cache:           15.47 GB/card
Available Memory:   15.31 GB/card
Total Used:         ~16.7 GB/card

KV Cache:           15.47 GB/card
CUDA Graphs:        6.11 GB/card
Available Memory:   9.20 GB/card
Total Used:         ~22.8 GB/card (在 32GB HBM2e 范围内 ✅)

curl http://localhost:8000/v1/models

curl http://localhost:8000/v1/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "glm-5-w4a8",
    "prompt": "你好，请介绍一下北京。",
    "max_tokens": 100,
    "temperature": 0.7
  }'

北京，位于华北平原，自古便是燕云十六州之一，是重要的军事要塞，具有重要的战略地位。

北京也是五朝帝都，分别为辽陪都、金中都、元大都、明清国都。

1949年10月1日，中华人民共和国成立，北京从此成为新中国的首都，成为全国的政治、经济、交通、文化中心。

北京是一座有着三千多年历史的古都，历史文化底蕴深厚，是全球拥有世界遗产数最多的城市。

chmod +x deploy_glm5_sglang_with_cudagraph.sh
bash deploy_glm5_sglang_with_cudagraph.sh > glm5_cudagraph.log 2>&1 &
tail -f glm5_cudagraph.log

bash deploy_glm5_sglang_deepep_optimized.sh > glm5_sglang.log 2>&1 &

--chunked-prefill-size 8192      # 不能超过 8192
--max-prefill-tokens 8192        # 不能超过 8192
--deepep-mode low_latency        # 推理优化模式

--chunked-prefill-size 16384     # 超限 2 倍
--max-prefill-tokens 280000      # 超限 35 倍！
--deepep-mode auto              # 未明确指定

--moe-a2a-backend deepep
--deepep-mode low_latency        # ✅ 唯一可用模式

--quantization modelslim

--tp-size 16                    # 16 卡张量并行
--mem-fraction-static 0.7       # 静态显存分配 70%
--disable-cuda-graph            # 禁用 CUDA Graph（NPU 不支持）
--skip-server-warmup            # 跳过预热，避免启动时算子错误
--trust-remote-code             # 信任模型代码

--max-prefill-tokens 280000  # 超限 35 倍
--moe-a2a-backend deepep
--deepep-mode auto

RuntimeError: call aclnnNotifyDispatchA2 failed, detail:EZ9999: Inner Error!
EZ9999[PID: 202188] NotifyDispatchA2 do tiling failed, ret is -1.
TraceBack (most recent call last):
   Check NnopbaseExecutorDoTiling(executor) failed
   Check NnopbaseExecutorTilingAndUpdateBinInfo(executor) failed
   Check NnopbaseExecutorMatchCache(executor) failed
   Check NnopbaseRunForWorkspace(*executor, workspaceSize) failed

--max-prefill-tokens 280000  # 超限
--moe-a2a-backend none       # 禁用 DeepEP

输入: "你好，请用一句话介绍北京"
输出: "变小fratanjzzaanschScreenshot multimEntitiesanis.ct carcteren-flatPerm..."

--moe-a2a-backend ascend_fuseep

RuntimeError: aclnnFusedDeepMoe or aclnnFusedDeepMoeGetWorkspaceSize not in libopapi.so, 
or libopapi.so not found.

--max-prefill-tokens 8192     # ✅ 符合硬件限制
--chunked-prefill-size 8192   # ✅ 符合硬件限制
--moe-a2a-backend deepep
--deepep-mode normal          # ❌ 此模式不可用

RuntimeError: call aclnnNotifyDispatchA2 failed, detail:EZ9999: Inner Error!
EZ9999[PID: 3259124] NotifyDispatchA2 do tiling failed, ret is -1.
   Check NnopbaseExecutorDoTiling(executor) failed

grep "max_prefill_tokens\|chunked_prefill_size" glm5_sglang.log

curl -s http://localhost:8000/v1/completions \
  -H "Content-Type: application/json" \
  -d '{"model": "glm-5-w4a8", "prompt": "北京是中国的", "max_tokens": 20}' \

watch -n 1 npu-smi info

curl http://localhost:8000/v1/models

{
  "object": "list",
  "data": [{
    "id": "glm-5-w4a8",
    "object": "model",
    "owned_by": "sglang",
    "max_model_len": 202752
  }]
}

curl -s http://localhost:8000/v1/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "glm-5-w4a8",
    "prompt": "北京是中国的",
    "max_tokens": 30,
    "temperature": 0.7
  }' 

{
  "choices": [{
    "text": "首都，人口众多，在这样的大城市里，想要找一份工作是非常不容易的..."
  }]
}

curl -s http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "glm-5-w4a8",
    "messages": [{"role": "user", "content": "介绍一下人工智能"}],
    "max_tokens": 50
  }'

# 1. 检查日志
cat glm5_sglang.log

# 2. 检查 CANN 环境
echo $CANN_INSTALL_PATH

# 3. 检查 NPU 状态
npu-smi info

# 4. 检查端口占用
netstat -tulpn | grep 8000

# 确认启动参数
ps aux | grep sglang.launch_server

# 检查是否包含
--moe-a2a-backend deepep
--deepep-mode low_latency
--max-prefill-tokens 8192

aclnnNotifyDispatchA2 failed
or
aclnnFusedDeepMoe not in libopapi.so

RuntimeError: 0 active drivers ([]). There should only be one.
File ".../allocator_npu.py", line 63, in alloc_extend
  alloc_extend_kernel[(bs,)](...)

# 使用启用 CUDA Graph 的脚本
bash deploy_glm5_sglang_with_cudagraph.sh > glm5_cudagraph.log 2>&1 &

# 使用禁用 CUDA Graph 的脚本（默认）
bash deploy_glm5_sglang_deepep_optimized.sh > glm5_sglang.log 2>&1 &

# NPU 利用率和显存
npu-smi info

# 服务吞吐量
grep "throughput" glm5_sglang.log | tail -20

# 请求延迟
# 在日志中查找 Prefill/Decode 时间