Spaces:
Runtime error
Runtime error
| import os | |
| from huggingface_hub import CommitOperationAdd, create_commit, RepoUrl | |
| from huggingface_hub import EvalResult, ModelCard | |
| from huggingface_hub.repocard_data import eval_results_to_model_index | |
| import time | |
| from pytablewriter import MarkdownTableWriter | |
| import gradio as gr | |
| from openllm import get_json_format_data, get_datas | |
| import pandas as pd | |
| import traceback | |
| BOT_HF_TOKEN = os.getenv('BOT_HF_TOKEN') | |
| data = get_json_format_data() | |
| finished_models = get_datas(data) | |
| df = pd.DataFrame(finished_models) | |
| desc = """ | |
| This is an automated PR created with https://huggingface.co/spaces/eduagarcia-temp/portuguese-leaderboard-results-to-modelcard | |
| The purpose of this PR is to add evaluation results from the Open Portuguese LLM Leaderboard to your model card. | |
| If you encounter any issues, please report them to https://huggingface.co/spaces/eduagarcia-temp/portuguese-leaderboard-results-to-modelcard/discussions | |
| """ | |
| def search(df, value): | |
| result_df = df[df["Model Name"] == value] | |
| return result_df.iloc[0].to_dict() if not result_df.empty else None | |
| def get_details_url(repo): | |
| #author, model = repo.split("/") | |
| return f"https://huggingface.co/datasets/eduagarcia-temp/llm_pt_leaderboard_raw_results/tree/main/{repo}" | |
| def get_query_url(repo): | |
| return f"https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard?query={repo}" | |
| def get_task_summary(results): | |
| return { | |
| "ENEM": | |
| {"dataset_type":"eduagarcia/enem_challenge", | |
| "dataset_name":"ENEM Challenge (No Images)", | |
| "metric_type":"acc", | |
| "metric_value":results["ENEM"], | |
| "dataset_config": None, | |
| "dataset_split":"train", | |
| "dataset_revision":None, | |
| "dataset_args":{"num_few_shot": 3}, | |
| "metric_name":"accuracy" | |
| }, | |
| "BLUEX": | |
| {"dataset_type":"eduagarcia-temp/BLUEX_without_images", | |
| "dataset_name":"BLUEX (No Images)", | |
| "metric_type":"acc", | |
| "metric_value":results["BLUEX"], | |
| "dataset_config": None, | |
| "dataset_split":"train", | |
| "dataset_revision":None, | |
| "dataset_args":{"num_few_shot": 3}, | |
| "metric_name":"accuracy" | |
| }, | |
| "OAB Exams": | |
| {"dataset_type":"eduagarcia/oab_exams", | |
| "dataset_name":"OAB Exams", | |
| "metric_type":"acc", | |
| "metric_value":results["OAB Exams"], | |
| "dataset_config": None, | |
| "dataset_split":"train", | |
| "dataset_revision":None, | |
| "dataset_args":{"num_few_shot": 3}, | |
| "metric_name":"accuracy" | |
| }, | |
| "ASSIN2 RTE": | |
| {"dataset_type":"assin2", | |
| "dataset_name":"Assin2 RTE", | |
| "metric_type":"f1_macro", | |
| "metric_value":results["ASSIN2 RTE"], | |
| "dataset_config": None, | |
| "dataset_split":"test", | |
| "dataset_revision":None, | |
| "dataset_args":{"num_few_shot": 15}, | |
| "metric_name":"f1-macro" | |
| }, | |
| "ASSIN2 STS": | |
| {"dataset_type":"assin2", | |
| "dataset_name":"Assin2 STS", | |
| "metric_type":"pearson", | |
| "metric_value":results["ASSIN2 STS"], | |
| "dataset_config": None, | |
| "dataset_split":"test", | |
| "dataset_revision":None, | |
| "dataset_args":{"num_few_shot": 15}, | |
| "metric_name":"pearson" | |
| }, | |
| "FAQUAD NLI": | |
| {"dataset_type":"ruanchaves/faquad-nli", | |
| "dataset_name":"FaQuAD NLI", | |
| "metric_type":"f1_macro", | |
| "metric_value":results["FAQUAD NLI"], | |
| "dataset_config": None, | |
| "dataset_split":"test", | |
| "dataset_revision":None, | |
| "dataset_args":{"num_few_shot": 15}, | |
| "metric_name":"f1-macro" | |
| }, | |
| "HateBR": | |
| {"dataset_type":"eduagarcia/portuguese_benchmark", | |
| "dataset_name":"HateBR Binary", | |
| "metric_type":"f1_macro", | |
| "metric_value":results["HateBR"], | |
| "dataset_config": None, | |
| "dataset_split":"test", | |
| "dataset_revision":None, | |
| "dataset_args":{"num_few_shot": 25}, | |
| "metric_name":"f1-macro" | |
| }, | |
| "PT Hate Speech": | |
| {"dataset_type":"eduagarcia/portuguese_benchmark", | |
| "dataset_name":"PT Hate Speech Binary", | |
| "metric_type":"f1_macro", | |
| "metric_value":results["PT Hate Speech"], | |
| "dataset_config": None, | |
| "dataset_split":"test", | |
| "dataset_revision":None, | |
| "dataset_args":{"num_few_shot": 25}, | |
| "metric_name":"f1-macro" | |
| }, | |
| "tweetSentBR": | |
| {"dataset_type":"eduagarcia-temp/tweetsentbr", | |
| "dataset_name":"tweetSentBR", | |
| "metric_type":"f1_macro", | |
| "metric_value":results["tweetSentBR"], | |
| "dataset_config": None, | |
| "dataset_split":"test", | |
| "dataset_revision":None, | |
| "dataset_args":{"num_few_shot": 25}, | |
| "metric_name":"f1-macro" | |
| } | |
| } | |
| def get_eval_results(repo): | |
| results = search(df, repo) | |
| task_summary = get_task_summary(results) | |
| md_writer = MarkdownTableWriter() | |
| md_writer.headers = ["Metric", "Value"] | |
| md_writer.value_matrix = [["Average", f"**{results['Average ⬆️']}**"]] + [[v["dataset_name"], v["metric_value"]] for v in task_summary.values()] | |
| text = f""" | |
| # [Open Portuguese LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard) | |
| Detailed results can be found [here]({get_details_url(repo)}) | |
| {md_writer.dumps()} | |
| """ | |
| return text | |
| def get_edited_yaml_readme(repo, token: str | None): | |
| card = ModelCard.load(repo, token=token) | |
| results = search(df, repo) | |
| common = {"task_type": 'text-generation', "task_name": 'Text Generation', "source_name": "Open Portuguese LLM Leaderboard", "source_url": get_query_url(repo)} | |
| tasks_results = get_task_summary(results) | |
| if not card.data['eval_results']: # No results reported yet, we initialize the metadata | |
| card.data["model-index"] = eval_results_to_model_index(repo.split('/')[1], [EvalResult(**task, **common) for task in tasks_results.values()]) | |
| else: # We add the new evaluations | |
| for task in tasks_results.values(): | |
| cur_result = EvalResult(**task, **common) | |
| if any(result.is_equal_except_value(cur_result) for result in card.data['eval_results']): | |
| continue | |
| card.data['eval_results'].append(cur_result) | |
| return str(card) | |
| def commit(repo, pr_number=None, message="Adding Evaluation Results", oauth_token: gr.OAuthToken | None = None): # specify pr number if you want to edit it, don't if you don't want | |
| if oauth_token is None: | |
| gr.Warning("You are not logged in; therefore, the leaderboard-pr-bot will open the pull request instead of you. Click on 'Sign in with Huggingface' to log in.") | |
| token = BOT_HF_TOKEN | |
| elif oauth_token.expires_at < time.time(): | |
| raise gr.Error("Token expired. Logout and try again.") | |
| else: | |
| token = oauth_token.token | |
| if repo.startswith("https://huggingface.co/"): | |
| try: | |
| repo = RepoUrl(repo).repo_id | |
| except Exception: | |
| raise gr.Error(f"Not a valid repo id: {str(repo)}") | |
| edited = {"revision": f"refs/pr/{pr_number}"} if pr_number else {"create_pr": True} | |
| try: | |
| try: # check if there is a readme already | |
| readme_text = get_edited_yaml_readme(repo, token=token) + get_eval_results(repo) | |
| except Exception as e: | |
| if "Repo card metadata block was not found." in str(e): # There is no readme | |
| readme_text = get_edited_yaml_readme(repo, token=token) | |
| else: | |
| traceback.print_exc() | |
| print(f"Something went wrong: {e}") | |
| liste = [CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme_text.encode())] | |
| commit = (create_commit(repo_id=repo, token=token, operations=liste, commit_message=message, commit_description=desc, repo_type="model", **edited).pr_url) | |
| return commit | |
| except Exception as e: | |
| if "Discussions are disabled for this repo" in str(e): | |
| return "Discussions disabled" | |
| elif "Cannot access gated repo" in str(e): | |
| return "Gated repo" | |
| elif "Repository Not Found" in str(e): | |
| return "Repository Not Found" | |
| else: | |
| return e | |
| if __name__ == "__main__": | |
| print(get_eval_results("Qwen/Qwen1.5-72B-Chat")) |