import logging import pandas as pd import os import csv import src.envs as envs from src.backend.model_operations import SummaryGenerator, EvaluationModel import src.backend.util as util logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') class Evaluator: """A class to evaluate summaries generated by a language model. Attributes: model (str): The name or path of the model. revision (str): The model revision. precision (str): The precision setting of the model. num_fewshot (int): Number of few-shot examples to use. batch_size (int): Batch size for processing. device (str): The device to run the model on. no_cache (bool): Flag to disable caching. limit (int): Limit on the number of items to process. write_out (bool): Whether to write results to a file. output_base_path (str): Base path for output files. summary_generator (SummaryGenerator): Instance for generating summaries. eval_model (EvaluationModel): Instance for evaluating summaries. """ def __init__(self, model, revision, precision, batch_size, device, no_cache, limit, write_out=True, output_base_path='logs'): """Initializes the Evaluator with the given model and settings. Args: model (str): The name or path of the model. revision (str): The model revision. precision (str): The precision setting of the model. num_fewshot (int): Number of few-shot examples to use. batch_size (int): Batch size for processing. device (str): The device to run the model on. no_cache (bool): Flag to disable caching. limit (int): Limit on the number of items to process. write_out (bool): Whether to write results to a file. output_base_path (str): Base path for output files. """ self.model = model self.revision = revision self.precision = precision self.batch_size = batch_size self.device = device self.no_cache = no_cache self.limit = limit self.write_out = write_out self.output_base_path = output_base_path try: self.summary_generator = SummaryGenerator(model, revision) self.eval_model = EvaluationModel(envs.HEM_PATH) except Exception as e: logging.error(f"Error initializing Evaluator: {e}") raise def evaluate(self): """ Performs the evaluation process by generating summaries and computing metrics. Returns: dict: A dictionary containing evaluation results. """ try: from openpyxl import load_workbook # df = load_workbook(filename=envs.DATASET_PATH) df_prompt = load_workbook(filename=envs.PROMPT_PATH) # df = pd.read_excel(envs.DATASET_PATH, engine='xlrd') #读取原数据，原始数据，本项目这里应该是问题 # df_prompt = pd.read_excel(envs.PROMPT_PATH, engine='xlrd') # df_prompt = pd.read_csv(envs.PROMPT_PATH) # print(envs.DATASET_PATH) # print(df.shape) # print(df.iloc[-1]) self.generated_summaries_df = self.summary_generator.generate_summaries(envs.DATASET_PATH, df_prompt, save_path=f"./generation_results/{self.model}.csv") # exit() # avg_summary_len = self.summary_generator.avg_length # answer_rate = self.summary_generator.answer_rate envs.API.upload_file( path_or_fileobj=f"./generation_results/{self.model}.csv", path_in_repo=f"{self.model}.csv", repo_id=envs.RESULTS_REPO, repo_type="dataset", ) '''开始评估模型的结果''' self.humanlike = self.eval_model.evaluate_humanlike(self.generated_summaries_df, envs.HUMAN_DATA, f"./generation_results/{self.model}.csv") all_results = self.humanlike # Prepare individual experiment scores and CIs experiment_results = {} for exp, data in all_results['per_experiment'].items(): experiment_results[f'{exp}'] = data['average_js_divergence'] experiment_results[f'{exp}_ci'] = data['confidence_interval'] # Write results into results using util.format_results results = util.format_results( model_name=self.model, revision=self.revision, precision=self.precision, overall_js=all_results['overall']['average_js_divergence'], overall_ci=all_results['overall']['confidence_interval'], **experiment_results # Unpack the experiment results ) '''原始指标''' # self.hallucination_scores, self.eval_results = self.eval_model.evaluate_hallucination( # self.generated_summaries_df) # factual_consistency_rate = self.eval_model.compute_factual_consistency_rate() # hallucination_rate = self.eval_model.hallucination_rate # factual_consistency_rate = 0 # answer_rate = 0 # avg_summary_len = 0 # # results = util.format_results(model_name=self.model, revision=self.revision, # precision=self.precision, # factual_consistency_rate=factual_consistency_rate, # hallucination_rate=self.humanlike, # answer_rate=answer_rate, # avg_summary_len=avg_summary_len) return results except FileNotFoundError: logging.error(f"File not found: {envs.DATASET_PATH}") raise except Exception as e: logging.error(f"Error during evaluation: {e}") raise def write_results(self): print('Updating result files') leaderboard_path = os.getcwd() # the path of leaderboard folder print(leaderboard_path) working_path = os.path.join(leaderboard_path, 'Humanlike Leaderboard Results') if not os.path.exists(working_path): logging.error(f"Need to first download the results from google drive to the learderboard folder") raise source_summary_df = self.generated_summaries_df[["user_prompt", "response"]] # #update leaderboard_summaries.csv # #first remove previous results for the current model # existing_df = pd.read_csv(os.path.join(working_path, 'leaderboard_summaries.csv'), encoding='utf-8', sep="\t") # mask = existing_df['model'] == self.model # existing_df = existing_df[~mask] # # get new result leaderboard_summaries_df = source_summary_df leaderboard_summaries_df.insert(2, "model", [self.model]*leaderboard_summaries_df.shape[0]) leaderboard_summaries_df.to_csv(os.path.join(working_path, 'leaderboard_summaries.csv'), mode='a', index=False, header=False) print('leaderboard_summaries.csv has been updated') # update leaderboard_summaries_with_scores.csv # BUG: get error when opening the file # existing_df = pd.read_csv(os.path.join(working_path, 'leaderboard_summaries_with_scores.csv'), # encoding='utf-8', sep=",", on_bad_lines='warn', quotechar='"', quoting=2) # print(existing_df.shape) # mask = existing_df['model'] == self.model # existing_df = existing_df[~mask] # get new result leaderboard_summaries_with_scores_df = pd.DataFrame.from_dict(self.eval_results) leaderboard_summaries_with_scores_df.insert(3, "model", [self.model]*leaderboard_summaries_with_scores_df.shape[0]) leaderboard_summaries_with_scores_df.to_csv(os.path.join(working_path, 'leaderboard_summaries_with_scores.csv'), mode='a', index=False, header=False) print('leaderboard_summaries_with_scores.csv has been updated')