ais_bench --models vllm_api_general_chat --datasets videobench_gen --debug --summarizer default_perf --mode perf --search

${AIS_HOME}/benchmark/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_chat.py

${AIS_HOME}/benchmark/ais_bench/benchmark/configs/datasets/videobench/videobench_gen.py

${AIS_HOME}/benchmark/ais_bench/datasets/synthetic/synthetic_config.py

from ais_bench.benchmark.models import VLLMCustomAPIChat
from ais_bench.benchmark.utils.model_postprocessors import extract_non_reasoning_content

models = [
    dict(
        attr="service",
        type=VLLMCustomAPIChat,
        abbr='vllm-api-general-chat',
        path="", #模型路径
        model="", #服务化模型名称
        request_rate = 0, 
        retry = 2,
        host_ip = "localhost", #服务化地址
        host_port = 8080, #服务化端口
        max_out_len = 512, #最大输出，根据性能测试要求配置
        batch_size=1, #并发数，根据性能测试要求配置
        trust_remote_code=False,
        generation_kwargs = dict( #后处理参数，根据性能测试要求配置
            temperature = 0.5,
            top_k = 10,
            top_p = 0.95,
            seed = None,
            repetition_penalty = 1.03,
        ),
        pred_postprocessor=dict(type=extract_non_reasoning_content)
    )
]

from ais_bench.benchmark.openicl.icl_prompt_template import PromptTemplate
from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever
from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer
from ais_bench.benchmark.datasets import VideoBenchDataset, VideoBenchEvaluator, math_postprocess_v2


videobench_reader_cfg = dict(
    input_columns=['question'],
    output_column='answer'
)


videobench_infer_cfg = dict(
    prompt_template=dict(
        type=PromptTemplate,
        template={'type': "video_text", 'data': ['video_url', 'text'],
                'prompt1': '\n Among the choice_length options choice_list above,'
                            ' the one closest to the correct answer is:',
                'prompt2': " Please respond with only the corresponding options and do not provide any explanations"
                        + " or additional information. ASSISTANT:"}
    ),
    retriever=dict(type=ZeroRetriever),
    inferencer=dict(type=GenInferencer)
)

videobench_eval_cfg = dict(
    evaluator=dict(type=VideoBenchEvaluator)
)

videobench_datasets = [
    dict(
        abbr='videobench',
        type=VideoBenchDataset,
        path='ais_bench/datasets/videobench', # 数据集路径，使用相对路径时相对于源码根路径，支持绝对路径,根据视频测试数据集路径修改
        reader_cfg=videobench_reader_cfg,
        infer_cfg=videobench_infer_cfg,
        eval_cfg=videobench_eval_cfg
    )
]

synthetic_config = {
    "Type":"string",   # [tokenid/string]，生成的随机数据集类型，支持固定长度的随机tokenid，和随机长度的string，两种类型的数据集
    "RequestCount": 256, # 生成的请求条数，应与模型侧配置文件中的 decode_batch_size 一致
    "TrustRemoteCode": False, #是否信任远端代码，tokenid模式下需要加载tokenizer生成tokenid，默认为Fasle
    "StringConfig" : {  # string类型的随机数据集的配置相关项，请参考以上注释处："StringConfig中的随机生成方法参数说明"
        "Input" : {     # 每条请求的输入长度
            "Method": "uniform",
            "Params": {"MinValue": 16384, "MaxValue": 16384} #性能测试输入，根据实际测试要求修改
        },
        "Output" : {    # 每条请求的输出长度
            "Method": "uniform",
            "Params": {"MinValue": 1024, "MaxValue": 1024} #性能测试输出，根据实际测试要求修改
        }
    },
}

ais_bench --models vllm_api_general_chat --datasets videobench_gen --debug --summarizer default_perf --mode perf