| """Plugin Evaluator.""" | |
| import json | |
| class TEvalEvaluator: | |
| """This module contains the following evaluators for evaluating the | |
| capabilities of the various dimensions of the LLM. | |
| specifically, InstructEvaluator is used to evaluate the instruction | |
| following capability of LLM, i.e. the ability of the model to perform tool | |
| calls according to an predefined format. ReasoningEvaluator is used to | |
| evaluate the model's ability to reason about the next execution step based | |
| on historical observations. PlanningEvaluator is used to evaluate the | |
| model's ability to plan a solution or program based on a given task. | |
| APIRetrievalEvaluator is used to evaluate the model's ability to retrieve a | |
| subset of tools relevant to the given task from a large number of tools. | |
| ReviewEvaluator is used to evaluate the model's ability to review whether a | |
| task was successfully completed. | |
| """ | |
| def __init__(self, subset) -> None: | |
| from opencompass.datasets.teval.evaluators import ( | |
| InstructEvaluator, PlanningEvaluator, | |
| ReasonRetrieveUnderstandEvaluator, ReviewEvaluator) | |
| super().__init__() | |
| self.subset = subset | |
| if subset == 'instruct': | |
| self.evaluator = InstructEvaluator('') | |
| elif subset == 'plan': | |
| self.evaluator = PlanningEvaluator('') | |
| elif subset == 'review': | |
| self.evaluator = ReviewEvaluator('') | |
| elif subset == 'reason_retrieve_understand': | |
| self.evaluator = ReasonRetrieveUnderstandEvaluator('') | |
| elif subset == 'reason': | |
| self.evaluator = ReasonRetrieveUnderstandEvaluator( | |
| '', default_prompt_type='str', eval_type='reason') | |
| elif subset == 'retrieve': | |
| self.evaluator = ReasonRetrieveUnderstandEvaluator( | |
| '', default_prompt_type='str', eval_type='retrieve') | |
| elif subset == 'understand': | |
| self.evaluator = ReasonRetrieveUnderstandEvaluator( | |
| '', default_prompt_type='str', eval_type='understand') | |
| elif subset == 'instruct_zh': | |
| self.evaluator = InstructEvaluator('') | |
| elif subset == 'plan_zh': | |
| self.evaluator = PlanningEvaluator( | |
| '', bert_score_model='thenlper/gte-large-zh') | |
| elif subset == 'review_zh': | |
| self.evaluator = ReviewEvaluator('') | |
| elif subset == 'reason_retrieve_understand_zh': | |
| self.evaluator = ReasonRetrieveUnderstandEvaluator( | |
| '', bert_score_model='thenlper/gte-large-zh') | |
| elif subset == 'reason_zh': | |
| self.evaluator = ReasonRetrieveUnderstandEvaluator( | |
| '', | |
| default_prompt_type='str', | |
| eval_type='reason', | |
| bert_score_model='thenlper/gte-large-zh') | |
| elif subset == 'retrieve_zh': | |
| self.evaluator = ReasonRetrieveUnderstandEvaluator( | |
| '', default_prompt_type='str', eval_type='retrieve') | |
| elif subset == 'understand_zh': | |
| self.evaluator = ReasonRetrieveUnderstandEvaluator( | |
| '', | |
| default_prompt_type='str', | |
| eval_type='understand', | |
| bert_score_model='thenlper/gte-large-zh') | |
| else: | |
| raise NotImplementedError | |
| def score(self, predictions, references): | |
| if len(predictions) != len(references): | |
| return { | |
| 'error': 'predictions and references have different ' | |
| 'length' | |
| } | |
| results_list = [] | |
| for prediction, reference in zip(predictions, references): | |
| datum = json.loads(reference) | |
| datum['prediction'] = prediction | |
| data_sample = self.evaluator._process_response(datum) | |
| if isinstance(data_sample, tuple): | |
| data_sample = data_sample[0] | |
| metrics_result = self.evaluator._evaluate(data_sample) | |
| results_list.append(metrics_result) | |
| results_dict = self.evaluator._post_process(results_list) | |
| results_dict = {k: v * 100 for k, v in results_dict.items()} | |
| return results_dict | |