File size: 11,050 Bytes
3ea18f8
b338d34
 
 
 
 
 
3ece82c
 
b338d34
3ea18f8
 
 
b338d34
3ea18f8
 
 
 
 
 
 
b338d34
3ea18f8
 
 
 
10974a0
3ea18f8
 
 
 
 
 
 
 
 
 
 
b338d34
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10974a0
3ece82c
 
10974a0
3ece82c
9c40f3c
328c0c0
3ece82c
 
 
 
 
 
9c40f3c
3ece82c
 
 
9c40f3c
3ece82c
 
 
b338d34
3ea18f8
b338d34
 
 
 
 
 
 
10974a0
b338d34
 
 
 
3ea18f8
b338d34
 
 
 
 
 
 
10974a0
3ea18f8
b338d34
3ea18f8
b338d34
3ea18f8
b338d34
 
 
fcc23ad
3ea18f8
 
b338d34
27cbb3d
 
 
 
 
 
 
10974a0
27cbb3d
3ece82c
3ea18f8
27cbb3d
1d13608
 
3ea18f8
ef3159e
328c0c0
3ece82c
328c0c0
3ea18f8
 
 
 
10974a0
 
328c0c0
10974a0
328c0c0
8753c00
328c0c0
8753c00
8d72f71
8753c00
8d72f71
10974a0
328c0c0
 
d265ff6
10974a0
 
 
 
 
 
 
 
 
d265ff6
 
 
 
 
 
 
 
 
0ebce0d
d265ff6
 
3ea18f8
d265ff6
3ea18f8
 
 
10974a0
 
 
 
 
 
 
 
 
 
 
 
 
 
27cbb3d
eb3a4e2
10974a0
27cbb3d
 
 
 
 
 
10974a0
27cbb3d
 
 
 
 
 
10974a0
 
27cbb3d
 
 
 
 
 
 
 
 
2167bbc
3ea18f8
7a236eb
3ea18f8
7a236eb
 
 
 
3ea18f8
2167bbc
3ea18f8
7a236eb
3ea18f8
2167bbc
3ea18f8
2167bbc
3ea18f8
7a236eb
 
3ea18f8
 
7a236eb
3ea18f8
 
b338d34
10974a0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
# Importing necessary libraries
import re
import streamlit as st
import requests
import pandas as pd
from io import StringIO
import plotly.graph_objs as go
from huggingface_hub import HfApi
from huggingface_hub.utils import RepositoryNotFoundError, RevisionNotFoundError
from yall import create_yall
from functools import cache



# Function to get model info from Hugging Face API using caching
@cache
def cached_model_info(api, model):
    try:
        return api.model_info(repo_id=str(model))
    except (RepositoryNotFoundError, RevisionNotFoundError):
        return None

# Function to get model info from DataFrame and update it with likes and tags
@st.cache
def get_model_info(df):
    api = HfApi()

    for index, row in df.iterrows():
        model_info = cached_model_info(api, row['Model'].strip())
        if model_info:
            df.loc[index, 'Likes'] = model_info.likes
            df.loc[index, 'Tags'] = ', '.join(model_info.tags)
        else:
            df.loc[index, 'Likes'] = -1
            df.loc[index, 'Tags'] = ''
    return df

# Function to convert markdown table to DataFrame and extract Hugging Face URLs
def convert_markdown_table_to_dataframe(md_content):
    """
    Converts markdown table to Pandas DataFrame, handling special characters and links,
    extracts Hugging Face URLs, and adds them to a new column.
    """
    # Remove leading and trailing | characters
    cleaned_content = re.sub(r'\|\s*$', '', re.sub(r'^\|\s*', '', md_content, flags=re.MULTILINE), flags=re.MULTILINE)

    # Create DataFrame from cleaned content
    df = pd.read_csv(StringIO(cleaned_content), sep="\|", engine='python')

    # Remove the first row after the header
    df = df.drop(0, axis=0)

    # Strip whitespace from column names
    df.columns = df.columns.str.strip()

    # Extract Hugging Face URLs and add them to a new column
    model_link_pattern = r'\[(.*?)\]\((.*?)\)\s*\[.*?\]\(.*?\)'
    df['URL'] = df['Model'].apply(lambda x: re.search(model_link_pattern, x).group(2) if re.search(model_link_pattern, x) else None)

    # Clean Model column to have only the model link text
    df['Model'] = df['Model'].apply(lambda x: re.sub(model_link_pattern, r'\1', x))

    return df

@st.cache_data
def get_model_info(df):
    api = HfApi()

    # Initialize new columns for likes and tags
    df['Likes'] = None
    df['Tags'] = None

    # Iterate through DataFrame rows
    for index, row in df.iterrows():
        model = row['Model'].strip()
        try:
            model_info = api.model_info(repo_id=str(model))
            df.loc[index, 'Likes'] = model_info.likes
            df.loc[index, 'Tags'] = ', '.join(model_info.tags)

        except (RepositoryNotFoundError, RevisionNotFoundError):
            df.loc[index, 'Likes'] = -1
            df.loc[index, 'Tags'] = ''

    return df

# Function to create bar chart for a given category
def create_bar_chart(df, category):
    """Create and display a bar chart for a given category."""
    st.write(f"### {category} Scores")

    # Sort the DataFrame based on the category score
    sorted_df = df[['Model', category]].sort_values(by=category, ascending=True)

    # Create the bar chart with a color gradient (using 'Viridis' color scale as an example)
    fig = go.Figure(go.Bar(
        x=sorted_df[category],
        y=sorted_df['Model'],
        orientation='h',
        marker=dict(color=sorted_df[category], colorscale='Spectral')  # You can change 'Viridis' to another color scale
    ))

    # Update layout for better readability
    fig.update_layout(
        margin=dict(l=20, r=20, t=20, b=20)
    )

    # Adjust the height of the chart based on the number of rows in the DataFrame
    st.plotly_chart(fig, use_container_width=True, height=len(df) * 35)

# Main function to run the Streamlit app
def main():
    # Set page configuration and title
    st.set_page_config(page_title="YALL - Yet Another LLM Leaderboard", layout="wide")

    st.title("πŸ† YALL - Yet Another LLM Leaderboard")
    st.markdown("Leaderboard made with 🧐 [LLM AutoEval](https://github.com/mlabonne/llm-autoeval) using [Nous](https://huggingface.co/NousResearch) benchmark suite.")

    # Create tabs for leaderboard and about section
    content = create_yall()
    tab1, tab2 = st.tabs(["πŸ† Leaderboard", "πŸ“ About"])

    # Leaderboard tab
    with tab1:
        if content:
            try:
                score_columns = ['Average', 'AGIEval', 'GPT4All', 'TruthfulQA', 'Bigbench']

                # Display dataframe
                full_df = convert_markdown_table_to_dataframe(content)

                for col in score_columns:
                    # Corrected use of pd.to_numeric
                    full_df[col] = pd.to_numeric(full_df[col].str.strip(), errors='coerce')

                full_df = get_model_info(full_df)
                full_df['Tags'] = full_df['Tags'].fillna('')
                df = pd.DataFrame(columns=full_df.columns)

                # Toggles for filtering by tags
                show_phi = st.checkbox("Phi (2.8B)", value=True)
                show_mistral = st.checkbox("Mistral (7B)", value=True)
                show_other = st.checkbox("Other", value=True)

                # Create a DataFrame based on selected filters
                dfs_to_concat = []

                if show_phi:
                    dfs_to_concat.append(full_df[full_df['Tags'].str.lower().str.contains('phi,|phi-msft,')])
                if show_mistral:
                    dfs_to_concat.append(full_df[full_df['Tags'].str.lower().str.contains('mistral,')])
                if show_other:
                    other_df = full_df[~full_df['Tags'].str.lower().str.contains('phi,|phi-msft,|mistral,')]
                    dfs_to_concat.append(other_df)

                # Concatenate the DataFrames
                if dfs_to_concat:
                    df = pd.concat(dfs_to_concat, ignore_index=True)

                # Add a search bar
                search_query = st.text_input("Search models", "")

                # Filter the DataFrame based on the search query
                if search_query:
                    df = df[df['Model'].str.contains(search_query, case=False)]

                # Display the filtered DataFrame or the entire leaderboard
                st.dataframe(
                    df[['Model'] + score_columns + ['Likes', 'URL']],
                    use_container_width=True,
                    column_config={
                        "Likes": st.column_config.NumberColumn(
                            "Likes",
                            help="Number of likes on Hugging Face",
                            format="%d ❀️",
                        ),
                        "URL": st.column_config.LinkColumn("URL"),
                    },
                    hide_index=True,
                    height=len(df) * 37,
                )
                selected_models = st.multiselect('Select models to compare', df['Model'].unique())
                comparison_df = df[df['Model'].isin(selected_models)]
                st.dataframe(comparison_df)
                # Add a button to export data to CSV
                if st.button("Export to CSV"):
                    # Export the DataFrame to CSV
                    csv_data = df.to_csv(index=False)

                    # Create a link to download the CSV file
                    st.download_button(
                        label="Download CSV",
                        data=csv_data,
                        file_name="leaderboard.csv",
                        key="download-csv",
                        help="Click to download the CSV file",
                    )

                # Full-width plot for the first category
                create_bar_chart(df, score_columns[0])

                # Next two plots in two columns
                col1, col2 = st.columns(2)
                with col1:
                    create_bar_chart(df, score_columns[1])
                with col2:
                    create_bar_chart(df, score_columns[2])

                # Last two plots in two columns
                col3, col4 = st.columns(2)
                with col3:
                    create_bar_chart(df, score_columns[3])
                with col4:
                    create_bar_chart(df, score_columns[4])


            except Exception as e:
                st.error("An error occurred while processing the markdown table.")
                st.error(str(e))
        else:
            st.error("Failed to download the content from the URL provided.")

     # About tab
    with tab2:
        st.markdown('''
            ### Nous benchmark suite

            Popularized by [Teknium](https://huggingface.co/teknium) and [NousResearch](https://huggingface.co/NousResearch), this benchmark suite aggregates four benchmarks:

            * [**AGIEval**](https://arxiv.org/abs/2304.06364) (0-shot): `agieval_aqua_rat,agieval_logiqa_en,agieval_lsat_ar,agieval_lsat_lr,agieval_lsat_rc,agieval_sat_en,agieval_sat_en_without_passage,agieval_sat_math`
            * **GPT4ALL** (0-shot): `hellaswag,openbookqa,winogrande,arc_easy,arc_challenge,boolq,piqa`
            * [**TruthfulQA**](https://arxiv.org/abs/2109.07958) (0-shot): `truthfulqa_mc`
            * [**Bigbench**](https://arxiv.org/abs/2206.04615) (0-shot): `bigbench_causal_judgement,bigbench_date_understanding,bigbench_disambiguation_qa,bigbench_geometric_shapes,bigbench_logical_deduction_five_objects,bigbench_logical_deduction_seven_objects,bigbench_logical_deduction_three_objects,bigbench_movie_recommendation,bigbench_navigate,bigbench_reasoning_about_colored_objects,bigbench_ruin_names,bigbench_salient_translation_error_detection,bigbench_snarks,bigbench_sports_understanding,bigbench_temporal_sequences,bigbench_tracking_shuffled_objects_five_objects,bigbench_tracking_shuffled_objects_seven_objects,bigbench_tracking_shuffled_objects_three_objects`

            ### Reproducibility

            You can easily reproduce these results using 🧐 [LLM AutoEval](https://github.com/mlabonne/llm-autoeval/tree/master), a colab notebook that automates the evaluation process (benchmark: `nous`). This will upload the results to GitHub as gists. You can find the entire table with the links to the detailed results [here](https://gist.github.com/mlabonne/90294929a2dbcb8877f9696f28105fdf).

            ### Clone this space

            You can create your own leaderboard with your LLM AutoEval results on GitHub Gist. You just need to clone this space and specify two variables:

            * Change the `gist_id` in [yall.py](https://huggingface.co/spaces/mlabonne/Yet_Another_LLM_Leaderboard/blob/main/yall.py#L126).
            * Create "New Secret" in Settings > Variables and secrets (name: "github", value: [your GitHub token](https://github.com/settings/tokens))

            A special thanks to [gblazex](https://huggingface.co/gblazex) for providing many evaluations.
        ''')

# Run the main function if this script is run directly
if __name__ == "__main__":
    main()