多模态模型通用能力评测及精度对齐

export IMAGE=m.daocloud.io/quay.io/ascend/vllm-ascend:v0.11.0
export container_name="qwen3vl8b"
export model_dir="/opt/data/modelscope/models"
export container_model_dir="/root/models"
export home_dir="/home"
export container_home_dir="/home"
docker run -itd --privileged \
--name $container_name \
--net=host \
--device /dev/davinci_manager \
--device /dev/devmm_svm \
--device /dev/hisi_hdc \
--shm-size=512g \
-v /usr/local/dcmi:/usr/local/dcmi \
-v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool \
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-v /etc/ascend_install.info:/etc/ascend_install.info \
-v ${model_dir}:${container_model_dir} \
-v ${home_dir}:${container_home_dir} \
$IMAGE /bin/bash

export VLLM_USE_V1=1; \
export VLLM_LOGGING_LEVEL=DEBUG; \
export ASCEND_RT_VISIBLE_DEVICES=3; \
export ts=`date "+%m%d%H%M%S"`; mkdir -p /home/outputs_vllm_qwen3vl_8b; \
nohup python -u /usr/local/python3.11.13/bin/vllm serve /root/models/Qwen/Qwen3-VL-8B-Instruct \
  --served-model-name qwen3-vl-8b \
  --host 0.0.0.0 \
  --port 33008 \
  --seed 1024 \
  --dtype bfloat16 \
  --max-num-seqs 8 \
  --tensor-parallel-size 1 \
  --max-model-len 36864 \
  --max-num-batched-tokens 8192 \
  --trust-remote-code \
  --gpu-memory-utilization 0.8 \
  --allowed-local-media-path /media_data \
  --no-enable-prefix-caching > /home/outputs_vllm_qwen3vl_8b/136-qwen3vl-8b_${ts}.log 2>&1 &
tail -f /home/outputs_vllm_qwen3vl_8b/136-qwen3vl-8b_${ts}.log

export VLLM_USE_V1=1;
export ts=`date "+%m%d%H%M%S"`; mkdir -p /root/outputs; \
nohup vllm serve /home/models/Qwen3-VL-8B-Instruct \
  --served-model-name qwen3-vl-8b \
  --host 0.0.0.0 \
  --port 8000 \
  --seed 1024 \
  --dtype bfloat16 \
  --max-num-seqs 8 \
  --tensor-parallel-size 4 \
  --max-model-len 34816 \
  --max-num-batched-tokens 8192 \
  --trust-remote-code \
  --gpu-memory-utilization 0.8 \
  --allowed-local-media-path / \
  --no-enable-prefix-caching > /home/136-qwen3vl-8b_${ts}.log 2>&1 &
tail -f /home/136-qwen3vl-8b_${ts}.log

def react_image_urls(obj):   # 增加打印过滤函数，避免打印媒体文件转码信息，同时尽量保留文本，方便定位
    """
    递归遍历 JSON 对象，将所有 image_url.url 的值替换为 "filtered"
    """
    if isinstance(obj, dict):
        # 如果当前字典包含 'image_url' 键，且其值是 dict 并含 'url'
        if 'image_url' in obj and isinstance(obj['image_url'], dict) and 'url' in obj['image_url']:
            obj['image_url']['url'] = 'filtered'
        # 递归处理所有子项
        for k, v in obj.items():
            redact_image_urls(v)
    elif isinstance(obj, list):
        for item in obj:
            redact_image_urls(item)

class OpenAIWrapper(BaseAPI):
……
    def generate_inner(self, inputs, **kwargs) -> str:   # 修改generate_inner函数增加api调用内容打印
……
        response = requests.post(
            self.api_base,
            headers=headers, data=json.dumps(payload), timeout=self.timeout * 1.1)
        ret_code = response.status_code
        ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code
        answer = self.fail_msg
        try:
            resp_struct = json.loads(response.text)

            payload_dump_content = copy.deepcopy(payload)
            react_image_urls(payload_dump_content)
            self.logger.warning(f'inputs={inputs}, self.api_base={self.api_base}, headers={headers}, payload={payload_dump_content}, '
                                f'timeout={self.timeout}, response.text={response.text}')   # 将输入和输出信息打印到一行，方便关联定位

            answer = resp_struct['choices'][0]['message']['content'].strip()
        except Exception as err:
            self.logger.error(f'{type(err)}: {err}')
            self.logger.error(response.text if hasattr(response, 'text') else response)

export model_dir="/opt/data/modelscope/models"
export container_model_dir="/root/models"
export container_name="Qwen3-VL"
export IMAGE=m.daocloud.io/quay.io/ascend/vllm-ascend:v0.11.0
docker run -itd --privileged \
--name $container_name \
--net=host \
--shm-size=1000g \
--device /dev/davinci0 \
--device /dev/davinci1 \
--device /dev/davinci2 \
--device /dev/davinci3 \
--device /dev/davinci4 \
--device /dev/davinci5 \
--device /dev/davinci6 \
--device /dev/davinci7 \
--device /dev/davinci8 \
--device /dev/davinci_manager \
--device /dev/devmm_svm \
--device /dev/hisi_hdc \
-v /usr/local/dcmi:/usr/local/dcmi \
-v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool \
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-v /etc/ascend_install.info:/etc/ascend_install.info \
-v ${model_dir}:${container_model_dir} \
$IMAGE /bin/bash

cd /path/to/Qwen3-VL/evaluation/mmmu
pip install -r requirements.txt

python run_mmmu.py infer \
    --model-path /root/models/Qwen/Qwen3-VL-30B-A3B-Instruct \
    --data-dir /root/data/AI-ModelScope/MMMU/ \
    --dataset MMMU_DEV_VAL \
    --output-file results/mmmu_dev_val_predictions.jsonl \
    --tensor-parallel-size 4 \
    --max-new-tokens 32768 \
    --temperature 0.7 \
    --top-p 0.8 \
    --top-k 20 \
    --repetition-penalty 1.0 \
    --presence-penalty 1.5

# max-model-len要设置大一些，有些推理结果超过了32768
nohup vllm serve /root/models/Qwen/Qwen3-30B-A3B-Instruct \
                --host 0.0.0.0 \
                --port 8000 \
                --tensor-parallel-size 4 \
                --max-num-seqs 2 \
                --max_num_batched_tokens 20000 \
                --trust-remote-code \
                --no-enable-prefix-caching \
                --gpu-memory-utilization 0.9 \
                --max-model-len 45000 \
                --dtype=bfloat16 \
                --served-model-name qwen3-30b > /root/qwen3-30b.log 2>&1 &

export MIT_SPIDER_TOKEN="test"
export MIT_SPIDER_URL="http://127.0.0.1:8000/v1/chat/completions"
python run_mmmu.py eval \
    --data-dir /root/data/AI-ModelScope/MMMU/ \
    --input-file results/mmmu_dev_val_predictions.jsonl \
    --output-file results/mmmu_dev_val_eval_results.csv \
    --dataset MMMU_DEV_VAL \
    --eval-model qwen3-30b \
    --api-type mit \
--nproc 16

export VLLM_USE_V1=1;
nohup vllm serve /home/models/Qwen3-VL-8B-Instruct \
  --served-model-name qwen3-vl-8b \
  --host 0.0.0.0 \
  --port 8000 \
  --seed 1024 \
  --dtype bfloat16 \
  --max-num-seqs 8 \
  --tensor-parallel-size 4 \
  --max-model-len 34816 \
  --max-num-batched-tokens 8192 \
  --trust-remote-code \
  --gpu-memory-utilization 0.8 \
  --allowed-local-media-path / \
  --no-enable-prefix-caching > /root/qwen3vl-8b.log 2>&1 &

#!/bin/bash

datasets=( "MMBench_DEV_EN_V11" "DocVQA_VAL" "OCRBench" "BLINK" "mmlu" )
api_url="http://71.10.29.136:60007/v1/chat/completions"
model=qwen3-vl-8b
nproc=4
limit=0
judge=chatgpt-0125
use_cache=no_use_cache
mode=all
infer_framework=vllmascend
log_dir="./outputs_${infer_framework}_${model}"

declare -A dataset_type
dataset_type["MMBench_DEV_EN_V11"]="vllm"
dataset_type["MMBench_DEV_CN_V11"]="vllm"
dataset_type["DocVQA_VAL"]="vllm"
dataset_type["InfoVQA_VAL"]="vllm"
dataset_type["ChartQA_TEST"]="vllm"
dataset_type["MMMU_DEV_VAL"]="vllm"
dataset_type["MMMU_Pro_10c"]="vllm"
dataset_type["MMMU_Pro_V"]="vllm"
dataset_type["MMMU_Pro_10c_COT"]="vllm"
dataset_type["MMMU_Pro_V_COT"]="vllm"
dataset_type["MathVision"]="vllm"
dataset_type["OCRBench"]="vllm"
dataset_type["BLINK"]="vllm"
dataset_type["Video-MME"]="vllm"
dataset_type["RefCOCO"]="vllm"
dataset_type["ifeval"]="llm"
dataset_type["gpqa"]="llm"
dataset_type["mmlu"]="llm"
dataset_type["mmlu_pro"]="llm"
dataset_type["ceval"]="llm"
dataset_type["hellaswag"]="llm"

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
}

log "Process datasets ${datasets[@]}"
mkdir -p ${log_dir}
for dataset in "${datasets[@]}"; do
    type="${dataset_type[$dataset]}"
    log "Running evaluation for dataset: ${dataset}, type: ${type}"

    ts=$(date "+%m%d%H%M%S")
    log_file="${log_dir}/batch_${infer_framework}_${model}_${dataset}_${nproc}_${limit}_${judge}_${ts}.log"
    args=(
        "./${infer_framework}_run_${type}_${model}.py"
        "$api_url"
        "$model"
        "$dataset"
        "$nproc"
        "$limit"
        "$judge"
        "$use_cache"
        "$mode"
    )

    printf -v full_command "python -u $(printf "%q " "${args[@]}")> %q 2>&1" "$log_file"
    log "${full_command}"
    python -u "${args[@]}" > "$log_file" 2>&1
    exit_code=$?
    if [ ${exit_code} -eq 0 ]; then
        log "Success proc ${dataset}. Log saved to: ${log_file}"
    else
        log "Failed proc ${dataset} with err ${exit_code}. Log saved to: ${log_file}"
    fi
    echo "----------------------------------------"
done

log "All datasets processed."
echo ""

from ruamel.yaml import YAML
from evalscope.run import run_task
from evalscope.summarizer import Summarizer
import time
import sys
import os

def replace_yml(config_path, url, model, dataset, nproc, limit, judge, use_cache, mode):
    try:
        yaml = YAML()
        yaml.preserve_quotes = True
        yaml.indent(mapping=2, sequence=4, offset=2)

        with open(config_path, 'r', encoding='utf-8') as file:
            config_data = yaml.load(file)

        if 'eval_config' in config_data and 'model' in config_data['eval_config']:
            if len(config_data['eval_config']['model']) > 0:
                config_data['eval_config']['model'][0]['type'] = model
                config_data['eval_config']['model'][0]['api_base'] = url

        if 'eval_config' in config_data and 'mode' in config_data['eval_config']:
            config_data['eval_config']['mode'] = mode

        if 'eval_config' in config_data and 'data' in config_data['eval_config']:
            if len(config_data['eval_config']['data']) > 0:
                config_data['eval_config']['data'][0] = dataset

        if 'eval_config' in config_data:
            config_data['eval_config']['nproc'] = nproc
            config_data['eval_config']['limit'] = limit
            config_data['eval_config']['judge'] = judge
            if use_cache == "no_use_cache":
                config_data['eval_config']['reuse'] = False
                config_data['use_cache'] = ""
            else:
                config_data['eval_config']['reuse'] = True
                config_data['use_cache'] = use_cache

        with open(config_path, 'w', encoding='utf-8') as file:
            yaml.dump(config_data, file)

        print(f"{config_path} update successful!")
        print(f"model={model}, url={url}, dataset={dataset}, nproc={nproc}, "
              f"limit={limit}, judge={judge}, use_cache={use_cache}, mode={mode}")
        return True

    except Exception as e:
        print(f"ERROR: {e}")
        return False

def run_vllm_eval(url, model, dataset, nproc, limit, judge, use_cache, mode):
    task_cfg = './vllmascend_run_vllm_qwen3-vl-8b.yaml'
    replace_yml(task_cfg, url, model, dataset, nproc, limit, judge, use_cache, mode)
    start_time = time.time()
    print('>> Start time is', start_time)
    run_task(task_cfg=task_cfg)
    print('>> Start to get the report with summarizer...')
    report_list = Summarizer.get_report_from_cfg(task_cfg)
    end_time = time.time()
    cost_time = end_time - start_time
    print(f'\n>> The report list: {report_list}, and cost {cost_time:.6f}s.')

if __name__ == '__main__':
    if len(sys.argv) <= 3:
        print('error: try use run_**.py {url} {model} {dataset} ${option_nproc_default32} ${option_limit_default0}\n')
        sys.exit(1)
    nproc = 32
    limit = 0
    judge = "exact_matching"
    use_cache = ""
    mode = "all"
    if len(sys.argv) > 3:
        url = sys.argv[1]
        model = sys.argv[2]
        dataset = sys.argv[3]
    if len(sys.argv) > 4:
        nproc = int(sys.argv[4])
    if len(sys.argv) > 5:
        limit = int(sys.argv[5])
    if len(sys.argv) > 6:
        judge = sys.argv[6]
    if len(sys.argv) > 7:
        use_cache = sys.argv[7]
    if len(sys.argv) > 8:
        mode = sys.argv[8]
    run_vllm_eval(url, model, dataset, nproc, limit, judge, use_cache, mode)

import sys
import time
from evalscope.run import run_task
from ruamel.yaml import YAML
from evalscope.summarizer import Summarizer
from evalscope.utils.logger import get_logger

logger = get_logger()

def replace_yml(config_path, url, model, dataset, nproc, limit, judge, use_cache, mode):
    try:
        yaml = YAML()
        yaml.preserve_quotes = True
        yaml.indent(mapping=2, sequence=4, offset=2)

        with open(config_path, 'r', encoding='utf-8') as file:
            config_data = yaml.load(file)

        if 'eval_config' in config_data and 'models' in config_data['eval_config']:
            if len(config_data['eval_config']['models']) > 0:
                config_data['eval_config']['models'][0]['path'] = model
                config_data['eval_config']['models'][0]['openai_api_base'] = url

        if 'eval_config' in config_data and 'mode' in config_data['eval_config']:
            config_data['eval_config']['mode'] = mode

        if 'eval_config' in config_data and 'datasets' in config_data['eval_config']:
            if len(config_data['eval_config']['datasets']) > 0:
                config_data['eval_config']['datasets'][0] = dataset

        if 'eval_config' in config_data:
            config_data['eval_config']['max_num_workers'] = nproc
            config_data['eval_config']['limit'] = limit

        with open(config_path, 'w', encoding='utf-8') as file:
            yaml.dump(config_data, file)

        logger.info(f"{config_path} update successful!")
        logger.info(f"model={model}, url={url}, dataset={dataset}, nproc={nproc}, "
                    f"limit={limit}, judge={judge}, use_cache={use_cache}, mode={mode}")
        return True

    except Exception as e:
        logger.info(f"ERROR: {e}")
        return False

def run_llm_eval(url, model, dataset, nproc, limit, judge, use_cache, mode):
    task_cfg = f'./vllmascend_run_llm_qwen3-vl-8b.yaml'
    replace_yml(task_cfg, url, model, dataset, nproc, limit, judge, use_cache, mode)

    run_task(task_cfg=task_cfg)
    print('>> Start to get the report with summarizer ...')
    report_list = Summarizer.get_report_from_cfg(task_cfg)
    print(f'\n>> The report list: {report_list}')


if __name__ == '__main__':
    if len(sys.argv) <= 3:
        logger.info('error: try use run_**.py {url} {model} {dataset} {option_nproc_default_32} {option_limit_default_0} {option_judge_default_exact_matching} {option_use_cache_default_null}\n')
        sys.exit(1)
    nproc = 1
    limit = 2
    judge = "exact_matching"
    use_cache = "no_use_cache"
    mode = "all"
    if len(sys.argv) > 3:
        url = sys.argv[1]
        model = sys.argv[2]
        dataset = sys.argv[3]
    if len(sys.argv) > 4:
        nproc = int(sys.argv[4])
    if len(sys.argv) > 5:
        limit = int(sys.argv[5])
    if len(sys.argv) > 6:
        judge = sys.argv[6]
    if len(sys.argv) > 7:
        use_cache = sys.argv[7]
    if len(sys.argv) > 8:
        mode = sys.argv[8]
    run_llm_eval(url, model, dataset, nproc, limit, judge, use_cache, mode)

work_dir: outputs_vllmascend_qwen3-vl-8b
eval_backend: VLMEvalKit
eval_config:
  model:
    - type: qwen3-vl-8b
      name: CustomAPIModel
      api_base: "http://71.10.29.136:33008/v1/chat/completions"
      key: EMPTY
      temperature: 0.7
      top_p: 0.8
      top_k: 20
      repetition_penalty: 1.0
      presence_penalty: 1.5
      img_size: -1
      img_detail: low
      timeout: 600
      retry: 1
      max_tokens: 16384

  data:
    - MMMU_DEV_VAL

  mode: all
  fps: -1
  nframe: 16
  limit: 2
  reuse: false
  nproc: 1
  verbose: true
  judge: chatgpt-0125
  OPENAI_API_KEY: EMPTY
  OPENAI_API_BASE: http://71.10.29.136:33030/v1/chat/completions # judge model api
  LOCAL_LLM: qwen3-vl-30b                            # judge model type

debug: true
use_cache: ''

model: text_generation
eval_backend: OpenCompass
eval_type: openai_api
work_dir: outputs_vllmascend_qwen3-vl-8b
eval_config:
  datasets:
    - gpqa
  models:
    - openai_api_base: http://71.10.29.136:33008/v1/chat/completions
      # type: OpenAI
      path: qwen3-vl-8b
      max_out_len: 16384
      max_seq_len: 32768
      temperature: 0.7
      top_p: 0.8
      top_k: 20
      repetition_penalty: 1.0
      presence_penalty: 1.5
      query_per_second: 1
      key: 'EMPTY'
      is_chat: true

  work_dir: outputs_vllmascend_pangu-vl-7b
  mode: all    # all, infer, eval, viz; must specify -r when running in eval or viz mode
  debug: true
  limit: 2
  max_num_workers: 1
debug: true
eval_batch_size: 1
timeout: 600

export ts=`date "+%m%d%H%M%S"`; export EVALSCOPE_LOG_LEVEL=DEBUG; \
nohup bash batch_run_benchmark.sh > /home/batch_run_benchmark_${ts}.log 2>&1 &
tail -f /home/batch_run_benchmark_${ts}.log