src/display/about.py

from dataclasses import dataclass
from enum import Enum

@dataclass
class Task:
    benchmark: str
    metric: str
    col_name: str


class Tasks(Enum):
    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
    Overall = Task("overall_js_divergence", "overall_js_divergence", "Overall Humanlike %")
    Overall_ci = Task("overall_confidence_interval", "overall_confidence_interval", "Overall CI")
    E1 = Task("E1", "E1", "Sound-1")
    # E1_ci = Task("E1_ci", "E1_ci", "Sound-1 CI")
    E2 = Task("E2", "E2", "Sound-2")
    # E2_ci = Task("E2_ci", "E2_ci", "Sound-2 CI")
    E3 = Task("E3", "E3", "Word-1")
    # E3_ci = Task("E3_ci", "E3_ci", "Word-1 CI")
    E4 = Task("E4", "E4", "Word-2")
    # E4_ci = Task("E4_ci", "E4_ci", "Word-2 CI")
    E5 = Task("E5", "E5", "Syntax-1")
    # E5_ci = Task("E5_ci", "E5_ci", "Syntax-1 CI")
    E6 = Task("E6", "E6", "Syntax-2")
    # E6_ci = Task("E6_ci", "E6_ci", "Syntax-2 CI")
    E7 = Task("E7", "E7", "Meaning-1")
    # E7_ci = Task("E7_ci", "E7_ci", "Meaning-1 CI")
    E8 = Task("E8", "E8", "Meaning-2")
    # E8_ci = Task("E8_ci", "E8_ci", "Meaning-2 CI")
    E9 = Task("E9", "E9", "Discourse-1")
    # E9_ci = Task("E9_ci", "E9_ci", "Discourse-1 CI")
    E10 = Task("E10", "E10", "Discourse-2")
    # E10_ci = Task("E10_ci", "E10_ci", "Discourse-2 CI")



# Your leaderboard name
TITLE = """<h1 align="center" id="space-title">Humanlike Evaluation Model (HEM) leaderboard</h1>"""

# What does your leaderboard evaluate?
INTRODUCTION_TEXT = """
Welcome to the Humanlikeness Leaderboard, curated by [Xufeng Duan](https://xufengduan.github.io/). This platform rigorously evaluates the alignment between human and model responses across five key aspects of language: sounds, words, syntax, meaning, and discourse, to quantify a model's humanlikeness.<br><br>
Data from 2000 human participants have been collected, creating response distributions that reflect natural human behavior. Identical stimuli are then presented to advanced language models, generating response distributions for comparison.<br><br>
The congruence between human and model responses provides a precise measure of the model's humanlikeness, offering critical insights into how closely these models mirror human cognitive processes.<br>
"""

# Which evaluations are you running? how can people reproduce what you have?
LLM_BENCHMARKS_TEXT = """
## Introduction

This study aims to compare the similarities between human and model responses in language use by employing ten psycholinguistic tasks:

1. **Sound:** Sound Shape Association<br>
2. **Sound:** Sound Gender Association<br>
3. **Word:** Word Length and Predictivity<br>
4. **Word:** Word Meaning Priming<br>
5. **Syntax:** Structural Priming<br>
6. **Syntax:** Syntactic Ambiguity Resolution<br>
7. **Meaning:** Implausible Sentence Interpretation<br>
8. **Meaning:** Semantic Illusion<br>
9. **Discourse:** Implicit Causality<br>
10. **Discourse:** Drawing Inferences<br><br>

Each task is composed of multiple stimuli, designed to elicit both expected and unexpected responses. We have gathered data from 2000 human participants, generating response distributions that reflect natural human behavior across these tasks. By presenting identical stimuli to advanced language models, we generate corresponding response distributions for comparison.<br><br>
The degree of congruence between these human and model distributions offers a precise measure of the model's humanlikeness.

## How it works

To measure the similarity between human and model responses, we utilize the Jensen-Shannon (JS) divergence. This method allows us to compare the two binomial distributions (one from human responses and one from model responses) for each stimulus. 
The similarity is quantified by calculating 1 minus the JS divergence, where a value closer to 1 indicates higher similarity.


## Model Submissions and Reproducibility
You can submit your model for evaluation, whether it's hosted on the Hugging Face model hub or not. (Though it is recommended to host your model on the Hugging Face)

### For models available on the Hugging Face model hub:
To replicate the evaluation result for a Hugging Face model:

1) Clone the Repository
```python
git lfs install
git clone https://huggingface.co/spaces/Simondon/HumanLikeness
```
2) Install the Requirements
```python
pip install -r requirements.txt
```
3) Set Up Your Hugging Face Token
```python
export HF_TOKEN=your_token
```
4) Run the Evaluation Script
```python
python main_backend.py --model your_model_id --precision float16
```
5) Check Results
After the evaluation, results are saved in "eval-results-bk/your_model_id/results.json".

## Results Format
The results are structured in JSON as follows:
```python
        {
          "config": {
            "model_dtype": "BF16",
            "model_name": "mistralai/Mistral-7B-Instruct-v0.1",
            "model_sha": ""
          },
          "results": {
            "overall_js_divergence": 0.6129438385008659,
            "overall_confidence_interval": [
              0.5937234777290732,
              0.6317188731175192
            ],
            "E9": 0.7768461816966632,
            "E9_ci": [
              0.7474754730701578,
              0.8058680968641126
            ],
            ...
          }
        }
```
For additional queries or model submissions, please contact [email protected].
"""

EVALUATION_QUEUE_TEXT = """

"""

CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
CITATION_BUTTON_TEXT = r"""
@dataset{HughesBae2023,
  author       = {Xufeng Duan, Bei Xiao, Xuemei Tang, Zhenguang Cai},
  title        = {Humanlike Leaderboard},
  year         = {2024},
  month        = {8},
  publisher    = {},
  doi          = {},
  url          = {https://huggingface.co/spaces/Simondon/HumanLikeness},
  abstract     = {A leaderboard comparing LLM performance at humanlikeness in language use.},
  keywords     = {nlp, llm, psycholinguistics, nli, machine learning},
  license      = {Apache-2.0},
}"""