Qwen3-VL-8B-Instruct-w8a8sc-310-vllm

python3 quant_qwen3vl.py \
    --model_path /path/to/qwen3_vl_float_weights \
    --save_directory /path/to/qwen3_vl_quantized_weights \
    --w_bit 4 \
    --a_bit 8 \
    --device_type npu \
    --anti_method m2 \
    --is_lowbit True \
    --fraction 0.01 \
    --use_sigma True \
    --torch_dtype fp16 \
    --trust_remote_code True

cd /usr/local/python3.11.10/lib/python3.11/site-packages/msmodelslim/pytorch/weight_compression/compress_graph/
# 编译weight_compression组件，其中${install_path}为CANN软件的安装目录。
bash build.sh ${install_path}/ascend-toolkit/latest
export LD_LIBRARY_PATH=/usr/local/python3.11.10/lib/:$LD_LIBRARY_PATH
python save_sharded_state_310.py \
    --model /path/to/qwen3_vl_quantized_weights \
    --tensor-parallel-size ${TP数} \
    --output /path/to/qwen3_vl_quantized_weights_used_on_310 \
    --enforce-eager \
    --dtype float16 \
    --quantization ascend \
    --max-model-len 16384 \
    --enable-compress \
    --compress-process-num 8