from dataclasses import dataclass from enum import Enum # @dataclass # class Ranking: # dimension: str # metric: str # col_name: str # class Rankings(Enum): # # dimension_key in the json file, metric_key in the json file, name to display in the leaderboard # rank0 = Ranking("overall", "Avg Score", "Overall") # rank1 = Ranking("math_algebra", "Avg Score", "Math (Algebra)") # rank2 = Ranking("math_geometry", "Avg Score", "Math (Geometry)") # rank3 = Ranking("math_prob", "Avg Score", "Math (Probability)") # rank4 = Ranking("reason_logical", "Avg Score", "Logical Reasoning") # rank5 = Ranking("reason_social", "Avg Score", "Social Reasoning") @dataclass class Domain: dimension: str metric: str col_name: str class Domains(Enum): # dimension_key in the json file, metric_key in the json file, name to display in the leaderboard dim0 = Domain("overall", "Avg Rank", "Overall") # dim1 = Task("math", "acc", "Math") # dim2 = Task("math_algebra", "acc", "Algebra") @dataclass class Task: benchmark: str metric: str col_name: str # Select your tasks here # --------------------------------------------------- class Tasks(Enum): # task_key in the json file, metric_key in the json file, name to display in the leaderboard task0 = Task("anli_r1", "acc", "ANLI") task1 = Task("logiqa", "acc_norm", "LogiQA") NUM_FEWSHOT = 0 # Change with your few shot # --------------------------------------------------- # Your leaderboard name TITLE = """

Decentralized Arena Leaderboard

""" SUB_TITLE = """

Building Automated, Robust, and Transparent LLM Evaluation for Numerous Dimensions

""" EXTERNAL_LINKS = """ """ # What does your leaderboard evaluate? INTRODUCTION_TEXT = """ **Decentralized Arena** automates and scales "Chatbot Arena" for LLM evaluation across various fine-grained dimensions (e.g., math – algebra, geometry, probability; logical reasoning, social reasoning, biology, chemistry, …). The evaluation is decentralized and democratic, with all LLMs participating in evaluating others. It achieves a 95\% correlation with Chatbot Arena's overall rankings, while being fully transparent and reproducible. """ # Which evaluations are you running? how can people reproduce what you have? LLM_BENCHMARKS_TEXT = f""" ## How it works ## Reproducibility To reproduce our results, here is the commands you can run: """ COMING_SOON_TEXT = """ # Coming soon We are working on adding more tasks to the leaderboard. Stay tuned! """ EVALUATION_QUEUE_TEXT = """ ## Some good practices before submitting a model ### 1) Make sure you can load your model and tokenizer using AutoClasses: ```python from transformers import AutoConfig, AutoModel, AutoTokenizer config = AutoConfig.from_pretrained("your model name", revision=revision) model = AutoModel.from_pretrained("your model name", revision=revision) tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision) ``` If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded. Note: make sure your model is public! Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted! ### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index) It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`! ### 3) Make sure your model has an open license! This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗 ### 4) Fill up your model card When we add extra information about models to the leaderboard, it will be automatically taken from the model card ## In case of model failure If your model is displayed in the `FAILED` category, its execution stopped. Make sure you have followed the above steps first. If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task). """ CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" CITATION_BUTTON_TEXT = r""" @misc{decentralized2024, title={Decentralized Arena via Collective LLM Intelligence: Building Automated, Robust, and Transparent LLM Evaluation for Numerous Dimensions}, author={Yanbin Yin, Zhen Wang, Kun Zhou, Xiangdong Zhang, Shibo Hao, Yi Gu, Jieyuan Liu, Somanshu Singla, Tianyang Liu, Eric P. Xing, Zhengzhong Liu, Haojian Jin, Zhiting Hu}, year=2024 } """