dataset_path: hrinnnn/PerceptionComp
dataset_kwargs:
  token: True
  cache_dir: perceptioncomp
  video: True
test_split: test
output_type: generate_until
doc_to_visual: !function utils.perceptioncomp_doc_to_visual
doc_to_text: !function utils.perceptioncomp_doc_to_text
doc_to_messages: !function utils.perceptioncomp_doc_to_messages
doc_to_target: "answer"
generation_kwargs:
  max_new_tokens: 64
  temperature: 0
  top_p: 1.0
  do_sample: false
process_results: !function utils.perceptioncomp_process_results
metric_list:
  - metric: perceptioncomp_accuracy
    aggregation: !function utils.perceptioncomp_aggregate_accuracy
    higher_is_better: true
  - metric: perceptioncomp_category_outdoor_tour
    aggregation: !function utils.perceptioncomp_aggregate_category_outdoor_tour
    higher_is_better: true
  - metric: perceptioncomp_category_shopping
    aggregation: !function utils.perceptioncomp_aggregate_category_shopping
    higher_is_better: true
  - metric: perceptioncomp_category_sport
    aggregation: !function utils.perceptioncomp_aggregate_category_sport
    higher_is_better: true
  - metric: perceptioncomp_category_variety_show
    aggregation: !function utils.perceptioncomp_aggregate_category_variety_show
    higher_is_better: true
  - metric: perceptioncomp_category_home_tour
    aggregation: !function utils.perceptioncomp_aggregate_category_home_tour
    higher_is_better: true
  - metric: perceptioncomp_category_game
    aggregation: !function utils.perceptioncomp_aggregate_category_game
    higher_is_better: true
  - metric: perceptioncomp_category_movie
    aggregation: !function utils.perceptioncomp_aggregate_category_movie
    higher_is_better: true
  - metric: perceptioncomp_difficulty_1
    aggregation: !function utils.perceptioncomp_aggregate_difficulty_1
    higher_is_better: true
  - metric: perceptioncomp_difficulty_2
    aggregation: !function utils.perceptioncomp_aggregate_difficulty_2
    higher_is_better: true
  - metric: perceptioncomp_difficulty_3
    aggregation: !function utils.perceptioncomp_aggregate_difficulty_3
    higher_is_better: true
lmms_eval_specific_kwargs:
  default:
    pre_prompt: ""
    post_prompt: "\nAnswer with the option's letter from the given choices directly."
  qwen3_vl:
    format: "qwen3_vl"
    pre_prompt: "Question: "
    post_prompt: "Answer with the option letter only."
metadata:
  - version: 0.0
