114 lines
4.4 KiB
Python
114 lines
4.4 KiB
Python
from collections import defaultdict
|
|
from typing import Any, Dict
|
|
|
|
from evalscope.benchmarks import Benchmark, DataAdapter
|
|
from evalscope.constants import EvalType, OutputType
|
|
from evalscope.metrics import exact_match
|
|
from evalscope.metrics.completion_parsers import ResponseParser
|
|
|
|
SUBSET_LIST = [
|
|
'computer science', 'math', 'chemistry', 'engineering', 'law', 'biology', 'health', 'physics', 'business',
|
|
'philosophy', 'economics', 'other', 'psychology', 'history'
|
|
]
|
|
|
|
|
|
@Benchmark.register(
|
|
name='mmlu_pro',
|
|
pretty_name='MMLU-Pro',
|
|
tags=['MCQ', 'Knowledge'],
|
|
description=
|
|
'MMLU-Pro is a benchmark for evaluating language models on multiple-choice questions across various subjects. It includes questions from different domains, where the model must select the correct answer from given options.', # noqa: E501
|
|
dataset_id='modelscope/MMLU-Pro',
|
|
model_adapter=OutputType.GENERATION,
|
|
output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
|
|
subset_list=SUBSET_LIST,
|
|
metric_list=['AverageAccuracy'],
|
|
few_shot_num=5,
|
|
train_split='validation',
|
|
eval_split='test',
|
|
prompt_template=
|
|
'The following are multiple choice questions (with answers) about {subset_name}. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n{query}', # noqa: E501
|
|
)
|
|
class MMLUProAdapter(DataAdapter):
|
|
|
|
def __init__(self, **kwargs):
|
|
super().__init__(**kwargs)
|
|
|
|
self.choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
|
|
|
|
def load(self, **kwargs):
|
|
# default load all data
|
|
kwargs['subset_list'] = ['default']
|
|
data_dict = super().load(**kwargs)
|
|
return self.reformat_subset(data_dict, subset_key='category')
|
|
|
|
def gen_prompt(self, input_d: Dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
|
|
if self.few_shot_num > 0:
|
|
prefix = self.format_fewshot_examples(few_shot_list)
|
|
else:
|
|
prefix = ''
|
|
query = prefix + 'Q: ' + input_d['question'] + '\n' + \
|
|
self.__form_options(input_d['options']) + '\n'
|
|
|
|
full_prompt = self.prompt_template.format(subset_name=subset_name, query=query)
|
|
return self.gen_prompt_data(full_prompt)
|
|
|
|
def format_fewshot_examples(self, few_shot_list):
|
|
# load few-shot prompts for each category
|
|
prompts = ''
|
|
for index, d in enumerate(few_shot_list):
|
|
prompts += 'Q: ' + d['question'] + '\n' + \
|
|
self.__form_options(d['options']) + '\n' + \
|
|
d['cot_content'] + '\n\n'
|
|
return prompts
|
|
|
|
def __form_options(self, options: list):
|
|
option_str = 'Options are:\n'
|
|
for opt, choice in zip(options, self.choices):
|
|
option_str += f'({choice}): {opt}' + '\n'
|
|
return option_str
|
|
|
|
def get_gold_answer(self, input_d: dict) -> str:
|
|
"""
|
|
Parse the raw input labels (gold).
|
|
|
|
Args:
|
|
input_d: input raw data. Depending on the dataset.
|
|
|
|
Returns:
|
|
The parsed input. e.g. gold answer ... Depending on the dataset.
|
|
"""
|
|
return input_d['answer']
|
|
|
|
def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
|
|
"""
|
|
Parse the predicted result and extract proper answer.
|
|
|
|
Args:
|
|
result: Predicted answer from the model. Usually a string for chat.
|
|
raw_input_d: The raw input. Depending on the dataset.
|
|
eval_type: 'checkpoint' or 'service' or `custom`, default: 'checkpoint'
|
|
|
|
Returns:
|
|
The parsed answer. Depending on the dataset. Usually a string for chat.
|
|
"""
|
|
if self.model_adapter == OutputType.MULTIPLE_CHOICE:
|
|
return result
|
|
else:
|
|
return ResponseParser.parse_first_option(result, options=self.choices)
|
|
|
|
def match(self, gold: str, pred: str) -> float:
|
|
"""
|
|
Match the gold answer and the predicted answer.
|
|
|
|
Args:
|
|
gold (Any): The golden answer. Usually a string for chat/multiple-choice-questions.
|
|
e.g. 'A', extracted from get_gold_answer method.
|
|
pred (Any): The predicted answer. Usually a string for chat/multiple-choice-questions.
|
|
e.g. 'B', extracted from parse_pred_result method.
|
|
|
|
Returns:
|
|
The match result. Usually a score (float) for chat/multiple-choice-questions.
|
|
"""
|
|
return exact_match(gold=gold, pred=pred)
|