Qwen3.6-27B INT4 AutoRound

vllm serve Lorbus/Qwen3.6-27B-int4-AutoRound \
  --dtype half \
  --max-model-len 262144 \
  --gpu-memory-utilization 0.85 \
  --kv-cache-dtype tq-t4nc \
  --max-num-seqs 3 \
  --reasoning-parser qwen3 \
  --enable-auto-tool-choice \
  --tool-call-parser qwen3_xml \
  --port 8888 --host 0.0.0.0 \
  --trust-remote-code \
  --compilation-config.cudagraph_mode none \
  --speculative-config '{"method": "mtp", "num_speculative_tokens": 1}'

from openai import OpenAI
client = OpenAI(base_url="http://localhost:8888/v1", api_key="EMPTY")
r = client.chat.completions.create(
    model="Lorbus/Qwen3.6-27B-int4-AutoRound",
    messages=[{"role": "user", "content": "Write a quicksort in Python."}],
    max_tokens=512,
)
print(r.choices[0].message.content)

from transformers import AutoModelForCausalLM, AutoTokenizer
m = AutoModelForCausalLM.from_pretrained(
    "Lorbus/Qwen3.6-27B-int4-AutoRound",
    trust_remote_code=True,
    device_map="auto",
)
tok = AutoTokenizer.from_pretrained("Lorbus/Qwen3.6-27B-int4-AutoRound")
msg = [{"role": "user", "content": "Explain quantum computing briefly."}]
ids = tok.apply_chat_template(msg, add_generation_prompt=True, return_tensors="pt").to(m.device)
print(tok.decode(m.generate(ids, max_new_tokens=256)[0]))

pip install auto-round-nightly

auto-round \
  --model Qwen/Qwen3.6-27B \
  --scheme W4A16 \
  --format auto_round \
  --output_dir Qwen3.6-27B-int4-AutoRound \
  --enable_torch_compile \
  --low_gpu_mem_usage \
  --device_map 0

#!/usr/bin/env python3
"""Dequantize mtp.fc from GPTQ INT4 back to bf16 so vLLM's MTP loader picks it up."""
import json, shutil
from pathlib import Path
import torch
from safetensors import safe_open
from safetensors.torch import save_file

BASE = Path("Qwen3.6-27B-int4-AutoRound")
EXTRA = BASE / "model_extra_tensors.safetensors"
INDEX = BASE / "model.safetensors.index.json"

tensors = {}
with safe_open(EXTRA, framework="pt") as f:
    meta = f.metadata() or {}
    for k in f.keys():
        tensors[k] = f.get_tensor(k)

qw = tensors["mtp.fc.qweight"]    # [1280, 5120] int32
qz = tensors["mtp.fc.qzeros"]     # [80, 640] int32
sc = tensors["mtp.fc.scales"]     # [80, 5120] fp16

in_features = qw.shape[0] * 8     # 10240
out_features = qw.shape[1]        # 5120
group_size = 128
num_groups = in_features // group_size   # 80

def unpack_int32_4bit(packed, axis, factor=8):
    dev = packed.device
    shifts = torch.arange(0, 32, 4, device=dev, dtype=torch.int32)
    expanded = (packed.unsqueeze(axis + 1) >> shifts.view([8 if i == axis + 1 else 1 for i in range(packed.ndim + 1)])) & 0xF
    new_shape = list(packed.shape); new_shape[axis] *= factor
    return expanded.reshape(new_shape).to(torch.int8)

w_int = unpack_int32_4bit(qw, axis=0)   # [10240, 5120]
z_int = unpack_int32_4bit(qz, axis=1)   # [80, 5120]

w_grouped = w_int.view(num_groups, group_size, out_features).to(torch.float32)
w_fp32 = (w_grouped - z_int.unsqueeze(1).to(torch.float32)) * sc.unsqueeze(1).to(torch.float32)
w_final = w_fp32.view(in_features, out_features).t().contiguous().to(torch.bfloat16)   # [5120, 10240]

# Replace
for k in ("mtp.fc.qweight", "mtp.fc.qzeros", "mtp.fc.scales"):
    del tensors[k]
tensors["mtp.fc.weight"] = w_final

save_file(tensors, str(EXTRA), metadata=meta)

# Update index
idx = json.loads(INDEX.read_text())
for k in ("mtp.fc.qweight", "mtp.fc.qzeros", "mtp.fc.scales"):
    idx["weight_map"].pop(k, None)
idx["weight_map"]["mtp.fc.weight"] = EXTRA.name
# Recompute total_size
from collections import defaultdict
shard_sizes = defaultdict(int)
for sf in set(idx["weight_map"].values()):
    with safe_open(BASE / sf, framework="pt") as f:
        for k in f.keys():
            t = f.get_tensor(k)
            shard_sizes[sf] += t.numel() * t.element_size()
idx["metadata"]["total_size"] = sum(shard_sizes.values())
INDEX.write_text(json.dumps(idx, indent=2))

@article{cheng2023autoround,
  title   = {Optimize Weight Rounding via Signed Gradient Descent for the Quantization of LLMs},
  author  = {Cheng, Wenhua and Zhang, Weiwei and Shen, Haihao and Cai, Yiyang and He, Xin and Lv, Kaokao and Liu, Yi},
  journal = {arXiv preprint arXiv:2309.05516},
  year    = {2023}
}