Qwen3-1.7B-FP16

W_fp16[out_idx, in_idx] = qweight_int8[out_idx, in_idx] × scales[in_idx // 128, out_idx]

Layer  0: norm=1.45 (嵌入层输出)
Layer  1: norm=153.72  ↑ 106×
Layer  5: norm=2399.63
Layer 10: norm=3166.80
Layer 20: norm=11301.76
Layer 27: norm=54710.95
Layer 28: norm=125.59  (final RMSNorm)

vllm serve /path/to/Qwen3-1.7B-FP16 \
  --enforce-eager \
  --max-model-len 4096 \
  --dtype float16 \
  --disable-log-stats \
  --port 18000 \
  --trust-remote-code \
  --gpu-memory-utilization 0.9

curl http://localhost:18000/v1/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "/path/to/Qwen3-1.7B-FP16",
    "prompt": "The capital of France is",
    "max_tokens": 20,
    "temperature": 0.6,
    "top_p": 0.95,
    "top_k": 20
  }'

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:18000/v1",
    api_key="not-needed",
)

response = client.chat.completions.create(
    model="/path/to/Qwen3-1.7B-FP16",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "What is the capital of France?"}
    ],
    max_tokens=50,
    temperature=0.6,
)

# convert_gptq_to_fp16.py
# 支持 GPTQ-Int8 -> FP16 的离线解量化

def dequantize_gptq_int8(
    qweight: torch.Tensor,    # 形状: (in_packed, out), dtype=int32
    scales: torch.Tensor,     # 形状: (num_groups, out), dtype=float16
    group_size: int = 128,    # GPTQ 分组大小
) -> torch.Tensor:
    in_packed, out_features = qweight.shape
    in_features = in_packed * 4
    
    # 解包 int32 -> int8
    qweight_t = qweight.T.contiguous()  # (out, in_packed)
    qweight_int8 = qweight_t.view(torch.int8)  # (out, in_features)
    
    # 扩展 scale: (num_groups, out) -> (out, in_features)
    num_groups = scales.shape[0]
    scales_expanded = torch.zeros(out_features, in_features, dtype=scales.dtype)
    for g in range(num_groups):
        start = g * group_size
        end = min(start + group_size, in_features)
        scales_expanded[:, start:end] = scales[g, :].unsqueeze(1)
    
    # 对称解量化: deq = qweight_int8 * scale
    dequantized = qweight_int8.to(scales.dtype) * scales_expanded
    
    return dequantized.contiguous()

# 1. 安装依赖
pip install torch safetensors transformers

# 2. 运行转换
python3 convert_gptq_to_fp16.py \
  /path/to/Qwen3-1.7B-GPTQ-Int8 \
  /path/to/Qwen3-1.7B-FP16

# 3. 验证 (可选)
python3 -c "
import safetensors.torch, torch
st = safetensors.torch.load_file('/path/to/Qwen3-1.7B-FP16/model.safetensors')
for k, v in st.items():
    if torch.isnan(v).any() or torch.isinf(v).any():
        print(f'WARNING: {k} has NaN/Inf!')
print(f'Loaded {len(st)} tensors, total {sum(t.numel()*t.element_size() for t in st.values())/1024/1024:.1f} MB')
"

@misc{qwen3technicalreport,
      title={Qwen3 Technical Report}, 
      author={Qwen Team},
      year={2025},
      eprint={2505.09388},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}