Qwen3-4B-FP8 — 昇腾 NPU 部署指南

# 安装 vLLM-Ascend
pip install vllm
pip install vllm-ascend

# 验证安装
python -c "import vllm; import vllm_ascend; print('OK')"

# 方式一：从 gitcode 克隆（推荐）
git lfs clone https://gitcode.com/2402_88120098/Qwen3_4B_FP8.git

# 方式二：从 HuggingFace 下载
git lfs clone https://huggingface.co/Qwen/Qwen3-4B-FP8

# 方式三：从 ModelScope 下载
pip install modelscope
python -c "from modelscope.hub.snapshot_download import snapshot_download; snapshot_download('Qwen/Qwen3-4B-FP8')"

MODEL_PATH=/path/to/Qwen3-4B-FP8

python -m vllm.entrypoints.openai.api_server \
    --model $MODEL_PATH \
    --max-model-len 32768 \
    --dtype auto \
    --gpu-memory-utilization 0.95 \
    --trust-remote-code \
    --port 8000

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="EMPTY"
)

# 非思考模式
response = client.chat.completions.create(
    model="/path/to/Qwen3-4B-FP8",
    messages=[{"role": "user", "content": "What is the capital of France?"}],
    temperature=0.0,
    max_tokens=64
)
print(response.choices[0].message.content)

# 思考模式（需 enable_thinking 参数）
response = client.chat.completions.create(
    model="/path/to/Qwen3-4B-FP8",
    messages=[{"role": "user", "content": "What is 15 × 27?"}],
    extra_body={"enable_thinking": True},
    temperature=0.0,
    max_tokens=256
)
print(response.choices[0].message.content)

The capital of France is Paris.

The play "Romeo and Juliet" was written by William Shakespeare.

The chemical symbol for water is H₂O, which consists of two hydrogen atoms and one oxygen atom.

...

The final answer is 405.

def is_even(n):
    return n % 2 == 0

# 通过 enable_thinking 参数切换
text = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True,
    enable_thinking=True  # True=思考模式, False=非思考模式
)

from transformers import AutoConfig, AutoModelForCausalLM

config = AutoConfig.from_pretrained("Qwen/Qwen3-4B-FP8")
# 启用 YaRN
config.rope_scaling = {
    "type": "yarn",
    "factor": 4.0,
    "original_max_position_embeddings": 32768
}
model = AutoModelForCausalLM.from_pretrained(
    "Qwen/Qwen3-4B-FP8",
    config=config,
    torch_dtype="auto"
)