本交付案例指导用户在 Ascend A2 集群上部署 DeepSeek-V4-Flash-0731-w8a8 模型。本次交付采用 2 Prefill + 4 Decode (2P4D) 的 PD 分离架构,并通过 MemCache 实现 KV Cache 跨节点复用,以最大化长上下文场景下的吞吐量与并发能力。
拓扑结构 (6 节点):
核心组件:
模型权重信息:
https://modelscope.cn/models/Eco-Tech/DeepSeek-V4-Flash-0731-w8a8
推理镜像信息:
docker pull quay.io/ascend/vllm-ascend:nightly-releases-v0.25.1rc
192.168.0.1192.168.0.2192.168.0.3192.168.0.4192.168.0.5192.168.0.6HDK 26.0.rc1 CANN 9.0
MemCache 需要一个中心化的 Meta 服务来协调所有节点的 Local 服务。在本案例中,Meta 服务部署在节点 1 上。
所有节点 (节点 1~节点6) 都需要配置 mmc-local.conf 文件,内容如下:
## 配置为MetaService启动节点IP
ock.mmc.meta_service_url = tcp://192.168.0.1:5000
## 配置为MetaService启动节点IP
ock.mmc.local_service.config_store_url = tcp://192.168.0.1:6000
## Log level: debug, info, warn, error
ock.mmc.log_level = info
## The maximum supported rank count; once ranks are connected, no further modifications are allowed — a meta restart is required
ock.mmc.local_service.world_size = 256
## 卡侧ROCE网络传输为A2典型配置
ock.mmc.local_service.protocol = device_rdma
## 每卡配置100GB内存容量,可根据环境实际内存大小适当调整
ock.mmc.local_service.dram.size = 200GB
ock.mmc.local_service.max.dram.size = 1024GB节点1 额外需要 mmc-meta.conf 文件来启动 Meta 服务:
## IP 需要修改为启动节点IP
ock.mmc.meta_service_url = tcp://192.168.0.1:5000
## IP 需要修改为启动节点IP
ock.mmc.meta_service.config_store_url = tcp://192.168.0.1:6000
## HTTP service url
ock.mmc.meta_service.metrics_url = http://192.168.0.1:7000
## Log level: debug, info, warn, error
ock.mmc.log_level = info步骤 1:配置 MemCache
/workspace/prefill-1/mmc-local.conf 内容如上节所示。/workspace/prefill-1/mmc-meta.conf 内容如上节所示。export MMC_META_CONFIG_PATH=/workspace/prefill-1/mmc-meta.conf
nohup python -c "from memcache_hybrid import MetaService; MetaService.main()" > memcache_meta.log 2>&1 &步骤 2:准备启动脚本
start_serve.sh:
#!/bin/bash
python launch_online_dp.py \
--dp-size 4 \
--tp-size 2 \
--dp-size-local 4 \
--dp-rank-start 0 \
--dp-address 192.168.0.1 \
--dp-rpc-port 12321 \
--vllm-start-port 7100run_dp_template.sh:
nic_name="bond1"
local_ip=192.168.0.1
# # jemalloc
export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD
# # AIV
export HCCL_OP_EXPANSION_MODE="AIV"
export TASK_QUEUE_ENABLE=1
export VLLM_API_KEY=sk-****
export MODEL_PATH="/data1/DeepSeek-V4-Flash-0731-w8a8"
export VLLM_RPC_TIMEOUT=3600000
export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=30000
export HCCL_EXEC_TIMEOUT=204
export HCCL_CONNECT_TIMEOUT=1200
export HCCL_IF_IP=$local_ip
export GLOO_SOCKET_IFNAME=$nic_name
export TP_SOCKET_IFNAME=$nic_name
export HCCL_SOCKET_IFNAME=$nic_name
export OMP_PROC_BIND=false
export OMP_NUM_THREADS=10
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export HCCL_BUFFSIZE=1024
export VLLM_ASCEND_APPLY_DSV4_PATCH=1
export ENABLE_PREFIX_CACHE=1
export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=4096
unset VLLM_ASCEND_ENABLE_FLASHCOMM1
export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD
export MMC_LOCAL_CONFIG_PATH=./mmc-local.conf
export PYTHONHASHSEED=0
export ASCEND_RT_VISIBLE_DEVICES=$1
nohup vllm serve "$MODEL_PATH" \
--host 0.0.0.0 \
--port $2 \
--api-key $VLLM_API_KEY \
--data-parallel-size $3 \
--data-parallel-rank $4 \
--data-parallel-address $5 \
--data-parallel-rpc-port $6 \
--tensor-parallel-size $7 \
--enable-expert-parallel \
--seed 1024 \
--served-model-name deepseek-v4-flash-0731-w8a8-292b \
--max-model-len 204800 \
--max-num-batched-tokens 4096 \
--max-num-seqs 32 \
--block-size 32 \
--enable-prefix-caching \
--enforce-eager \
--async-scheduling \
--no-disable-hybrid-kv-cache-manager \
--trust-remote-code \
--gpu-memory-utilization 0.95 \
--quantization ascend \
--model-loader-extra-config '{"enable_multithread_load":true,"num_threads":128}' \
--tokenizer-mode deepseek_v4 \
--tool-call-parser deepseek_v4 \
--enable-auto-tool-choice \
--reasoning-parser deepseek_v4 \
--additional-config '{"enable_cpu_binding":true,"enable_dsa_cp":true,"enable_flashcomm1":true}' \
--speculative-config '{"method":"dspark","num_speculative_tokens":7,"enforce_eager":true}' \
--kv-transfer-config '{"kv_connector":"MultiConnector","kv_role":"kv_producer","engine_id":"0","kv_connector_extra_config":{"connectors":[{"kv_connector":"MooncakeHybridConnector","kv_role":"kv_producer","kv_port":"30000","kv_connector_extra_config":{"use_ascend_direct":true,"prefill":{"dp_size":4,"tp_size":2},"decode":{"dp_size":4,"tp_size":2}}},{"kv_connector":"AscendStoreConnector","kv_role":"kv_producer","kv_connector_extra_config":{"backend":"memcache","lookup_rpc_port":"0"}}]}}' \
> prefill_1-$(date +%Y%m%d-%H%M%S).log 2>&1 &步骤 3:启动服务
cd /workspace/prefill-1
bash start_serve.sh此脚本会调用 launch_online_dp.py,该工具会根据参数自动调用 run_dp_template.sh 来启动 4 个 DP rank 的 vLLM 进程。
步骤 1:配置 MemCache
确保 /workspace/prefill-2/mmc-local.conf 内容如下:
## 配置为MetaService启动节点IP
ock.mmc.meta_service_url = tcp://192.168.0.1:5000
## 配置为MetaService启动节点IP
ock.mmc.local_service.config_store_url = tcp://192.168.0.1:6000
## Log level: debug, info, warn, error
ock.mmc.log_level = info
## The maximum supported rank count; once ranks are connected, no further modifications are allowed — a meta restart is required
ock.mmc.local_service.world_size = 256
## 卡侧ROCE网络传输为A2典型配置
ock.mmc.local_service.protocol = device_rdma
## 每卡配置100GB内存容量,可根据环境实际内存大小适当调整
ock.mmc.local_service.dram.size = 200GB
ock.mmc.local_service.max.dram.size = 1024GB步骤 2:准备 Prefill 2 启动脚本
start_serve.sh:
#!/bin/bash
python launch_online_dp.py \
--dp-size 4 \
--tp-size 2 \
--dp-size-local 4 \
--dp-rank-start 0 \
--dp-address 192.168.0.2 \
--dp-rpc-port 12321 \
--vllm-start-port 7100run_dp_template.sh:
nic_name="bond1"
local_ip=192.168.0.2
# # jemalloc
export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD
# # AIV
export HCCL_OP_EXPANSION_MODE="AIV"
export TASK_QUEUE_ENABLE=1
export VLLM_API_KEY=sk-****
export MODEL_PATH="/data1/DeepSeek-V4-Flash-0731-w8a8"
export VLLM_RPC_TIMEOUT=3600000
export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=30000
export HCCL_EXEC_TIMEOUT=204
export HCCL_CONNECT_TIMEOUT=1200
export HCCL_IF_IP=$local_ip
export GLOO_SOCKET_IFNAME=$nic_name
export TP_SOCKET_IFNAME=$nic_name
export HCCL_SOCKET_IFNAME=$nic_name
export OMP_PROC_BIND=false
export OMP_NUM_THREADS=10
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export HCCL_BUFFSIZE=1024
export VLLM_ASCEND_APPLY_DSV4_PATCH=1
export ENABLE_PREFIX_CACHE=1
export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=4096
unset VLLM_ASCEND_ENABLE_FLASHCOMM1
export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD
export MMC_LOCAL_CONFIG_PATH=./mmc-local.conf
export PYTHONHASHSEED=0
export ASCEND_RT_VISIBLE_DEVICES=$1
nohup vllm serve "$MODEL_PATH" \
--host 0.0.0.0 \
--port $2 \
--api-key $VLLM_API_KEY \
--data-parallel-size $3 \
--data-parallel-rank $4 \
--data-parallel-address $5 \
--data-parallel-rpc-port $6 \
--tensor-parallel-size $7 \
--enable-expert-parallel \
--seed 1024 \
--served-model-name deepseek-v4-flash-0731-w8a8-292b \
--max-model-len 204800 \
--max-num-batched-tokens 4096 \
--max-num-seqs 32 \
--block-size 32 \
--enable-prefix-caching \
--enforce-eager \
--async-scheduling \
--no-disable-hybrid-kv-cache-manager \
--trust-remote-code \
--gpu-memory-utilization 0.95 \
--quantization ascend \
--model-loader-extra-config '{"enable_multithread_load":true,"num_threads":128}' \
--tokenizer-mode deepseek_v4 \
--tool-call-parser deepseek_v4 \
--enable-auto-tool-choice \
--reasoning-parser deepseek_v4 \
--additional-config '{"enable_cpu_binding":true,"enable_dsa_cp":true,"enable_flashcomm1":true}' \
--speculative-config '{"method":"dspark","num_speculative_tokens":7,"enforce_eager":true}' \
--kv-transfer-config '{"kv_connector":"MultiConnector","kv_role":"kv_producer","engine_id":"1","kv_connector_extra_config":{"connectors":[{"kv_connector":"MooncakeHybridConnector","kv_role":"kv_producer","kv_port":"30100","kv_connector_extra_config":{"use_ascend_direct":true,"prefill":{"dp_size":4,"tp_size":2},"decode":{"dp_size":4,"tp_size":2}}},{"kv_connector":"AscendStoreConnector","kv_role":"kv_producer","kv_connector_extra_config":{"backend":"memcache","lookup_rpc_port":"1"}}]}}' \
> prefill_2-$(date +%Y%m%d-%H%M%S).log 2>&1 &步骤 3:启动 Prefill 2 服务
cd /workspace/prefill-2
bash start_serve.sh每个 Decode 节点配置类似,只需要修改 local_ip、engine_id、kv_port 和 lookup_rpc_port。
步骤 1:配置 MemCache
确保每个 Decode 节点的 /workspace/decode/mmc-local.conf 内容如下:
## 配置为MetaService启动节点IP
ock.mmc.meta_service_url = tcp://192.168.0.1:5000
## 配置为MetaService启动节点IP
ock.mmc.local_service.config_store_url = tcp://192.168.0.1:6000
## Log level: debug, info, warn, error
ock.mmc.log_level = info
## The maximum supported rank count; once ranks are connected, no further modifications are allowed — a meta restart is required
ock.mmc.local_service.world_size = 256
## 卡侧ROCE网络传输为A2典型配置
ock.mmc.local_service.protocol = device_rdma
## 每卡配置100GB内存容量,可根据环境实际内存大小适当调整
ock.mmc.local_service.dram.size = 200GB
ock.mmc.local_service.max.dram.size = 1024GB步骤 2:准备启动脚本
start_serve.sh: 内容类似,只需修改 --dp-address 为本机 IP。
#!/bin/bash
python launch_online_dp.py \
--dp-size 4 \
--tp-size 2 \
--dp-size-local 4 \
--dp-rank-start 0 \
--dp-address 192.168.0.3 \
--dp-rpc-port 12321 \
--vllm-start-port 7100#!/bin/bash
python launch_online_dp.py \
--dp-size 4 \
--tp-size 2 \
--dp-size-local 4 \
--dp-rank-start 0 \
--dp-address 192.168.0.4 \
--dp-rpc-port 12321 \
--vllm-start-port 7100#!/bin/bash
python launch_online_dp.py \
--dp-size 4 \
--tp-size 2 \
--dp-size-local 4 \
--dp-rank-start 0 \
--dp-address 192.168.0.5 \
--dp-rpc-port 12321 \
--vllm-start-port 7100#!/bin/bash
python launch_online_dp.py \
--dp-size 4 \
--tp-size 2 \
--dp-size-local 4 \
--dp-rank-start 0 \
--dp-address 192.168.0.6 \
--dp-rpc-port 12321 \
--vllm-start-port 7100run_dp_template.sh:
每个节点基本相同,但需要修改nic_name、 local_ip、engine_id、kv_port、lookup_rpc_port 和日志文件名。
节点3 (Decode 1):
nic_name="bond1"
local_ip=192.168.0.3
export VLLM_API_KEY=sk-****
export MODEL_PATH="/data1/DeepSeek-V4-Flash-0731-w8a8"
set +u
source /usr/local/Ascend/ascend-toolkit/set_env.sh
set -u
export LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:${LD_PRELOAD:-}"
export HCCL_OP_EXPANSION_MODE=AIV
export TASK_QUEUE_ENABLE=1
export VLLM_ASCEND_APPLY_DSV4_PATCH=1
export VLLM_RPC_TIMEOUT=3600000
export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=30000
export HCCL_EXEC_TIMEOUT=204
export HCCL_CONNECT_TIMEOUT=1200
export HCCL_IF_IP=$local_ip
export GLOO_SOCKET_IFNAME=$nic_name
export TP_SOCKET_IFNAME=$nic_name
export HCCL_SOCKET_IFNAME=$nic_name
export OMP_PROC_BIND=false
export OMP_NUM_THREADS=10
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export HCCL_BUFFSIZE=1024
export ASCEND_RT_VISIBLE_DEVICES=$1
export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD
export MMC_LOCAL_CONFIG_PATH=./mmc-local.conf
export PYTHONHASHSEED=0
exec vllm serve "$MODEL_PATH" \
--host 0.0.0.0 \
--port $2 \
--api-key $VLLM_API_KEY \
--data-parallel-size $3 \
--data-parallel-rank $4 \
--data-parallel-address $5 \
--data-parallel-rpc-port $6 \
--tensor-parallel-size $7 \
--enable-expert-parallel \
--seed 1024 \
--served-model-name deepseek-v4-flash-0731-w8a8-292b \
--max-model-len 204800 \
--max-num-batched-tokens 256 \
--max-num-seqs 32 \
--async-scheduling \
--block-size 32 \
--no-disable-hybrid-kv-cache-manager \
--trust-remote-code \
--gpu-memory-utilization 0.9 \
--quantization ascend \
--model-loader-extra-config '{"enable_multithread_load":true,"num_threads":128}' \
--tokenizer-mode deepseek_v4 \
--tool-call-parser deepseek_v4 \
--enable-auto-tool-choice \
--reasoning-parser deepseek_v4 \
--speculative-config '{"method":"dspark","num_speculative_tokens":3,"enforce_eager":true}' \
--compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}' \
--kv-transfer-config '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","engine_id":"2","kv_connector_extra_config":{"connectors":[{"kv_connector":"MooncakeHybridConnector","kv_role":"kv_consumer","kv_port":"30200","kv_connector_extra_config":{"use_ascend_direct":true,"prefill":{"dp_size":4,"tp_size":2},"decode":{"dp_size":4,"tp_size":2}}},{"kv_connector":"AscendStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"backend":"memcache","lookup_rpc_port":"2"}}]}}' \
--additional-config '{"ascend_compilation_config":{"enable_npugraph_ex":true,"enable_static_kernel":false},"enable_cpu_binding":true,"multistream_overlap_shared_expert":true}' \
> decode_1-$(date +%Y%m%d-%H%M%S).log 2>&1 &节点4 (Decode 2):
# ... 同上 ...
local_ip=192.168.0.4
# ... 中间部分相同 ...
--kv-transfer-config '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","engine_id":"3","kv_connector_extra_config":{"connectors":[{"kv_connector":"MooncakeHybridConnector","kv_role":"kv_consumer","kv_port":"30300","kv_connector_extra_config":{"use_ascend_direct":true,"prefill":{"dp_size":4,"tp_size":2},"decode":{"dp_size":4,"tp_size":2}}},{"kv_connector":"AscendStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"backend":"memcache","lookup_rpc_port":"3"}}]}}' \
> decode_1-$(date +%Y%m%d-%H%M%S).log 2>&1 &节点5 (Decode 3):
# ... 同上 ...
local_ip=192.168.0.5
# ... 中间部分相同 ...
--kv-transfer-config '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","engine_id":"4","kv_connector_extra_config":{"connectors":[{"kv_connector":"MooncakeHybridConnector","kv_role":"kv_consumer","kv_port":"30400","kv_connector_extra_config":{"use_ascend_direct":true,"prefill":{"dp_size":4,"tp_size":2},"decode":{"dp_size":4,"tp_size":2}}},{"kv_connector":"AscendStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"backend":"memcache","lookup_rpc_port":"4"}}]}}' \
> decode_1-$(date +%Y%m%d-%H%M%S).log 2>&1 &节点6(Decode 4):
# ... 同上 ...
local_ip=192.168.0.6
# ... 中间部分相同 ...
--kv-transfer-config '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","engine_id":"5","kv_connector_extra_config":{"connectors":[{"kv_connector":"MooncakeHybridConnector","kv_role":"kv_consumer","kv_port":"30500","kv_connector_extra_config":{"use_ascend_direct":true,"prefill":{"dp_size":4,"tp_size":2},"decode":{"dp_size":4,"tp_size":2}}},{"kv_connector":"AscendStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"backend":"memcache","lookup_rpc_port":"5"}}]}}' \
> decode_1-$(date +%Y%m%d-%H%M%S).log 2>&1 &步骤 3:启动 Decode 服务
在每个 Decode 节点上执行:
cd /workspace/decode
bash start_serve.sh步骤 4:准备并启动 Proxy 服务
在节点 2 上创建 run_proxy.sh 脚本:
#!/bin/bash
PROXY_TYPE="basic"
PROXY_PORT="8000"
PROXY_HOST="192.168.0.2"
PREFILLER_HOSTS="192.168.0.1 192.168.0.1 192.168.0.1 192.168.0.1 192.168.0.2 192.168.0.2 192.168.0.2 192.168.0.2"
PREFILLER_PORTS="7100 7101 7102 7103 7100 7101 7102 7103"
DECODER_HOSTS="192.168.0.3 192.168.0.3 192.168.0.3 192.168.0.3 192.168.0.4 192.168.0.4 192.168.0.4 192.168.0.4 192.168.0.5 192.168.0.5 192.168.0.5 192.168.0.5 192.168.0.6 192.168.0.6 192.168.0.6 192.168.0.6 "
DECODER_PORTS="7100 7101 7102 7103 7100 7101 7102 7103 7100 7101 7102 7103 7100 7101 7102 7103"
if [ "$PROXY_TYPE" == "basic" ]; then
python3 load_balance_proxy_server_example.py \
--port $PROXY_PORT \
--host $PROXY_HOST \
--prefiller-hosts $PREFILLER_HOSTS \
--prefiller-ports $PREFILLER_PORTS \
--decoder-hosts $DECODER_HOSTS \
--decoder-ports $DECODER_PORTS
else
python3 load_balance_proxy_layerwise_server_example.py \
--port $PROXY_PORT \
--host $PROXY_HOST \
--prefiller-hosts $PREFILLER_HOSTS \
--prefiller-ports $PREFILLER_PORTS \
--decoder-hosts $DECODER_HOSTS \
--decoder-ports $DECODER_PORTS
fistart-proxy.sh
nohup bash run_proxy.sh > proxy-$(date +%Y%m%d-%H%M%S).log 2>&1 &启动 Proxy 服务:
bash start-proxy.sh通过 节点2 的 8000 端口访问:
curl -sS http://192.168.0.2:8000/v1/models
# 应返回模型列表,包含 deepseek-v4-flash-0731-w8a8-292b
curl -sS http://192.168.0.2:8000/v1/chat/completions \
-H 'Content-Type: application/json' \
-d '{
"model": "deepseek-v4-flash-0731-w8a8-292b",
"messages": [{"role": "user", "content": "杭州是哪个省的省会?"}],
"max_tokens": 10
}'
# 应返回正确结果 "浙江"kv_connector 初始化成功,无 RDMA 连接错误。