import os import gradio as gr import pandas as pd from apscheduler.schedulers.background import BackgroundScheduler from collections import Counter, defaultdict from datasets import load_dataset import datasets from huggingface_hub import HfApi, list_datasets api = HfApi(token=os.environ.get("HF_TOKEN", None)) def restart_space(): api.restart_space(repo_id="OpenGenAI/parti-prompts-leaderboard") parti_prompt_results = [] ORG = "diffusers-parti-prompts" SUBMISSIONS = { "kand2": None, "sdxl": None, "wuerst": None, "karlo": None, } LINKS = { "kand2": "https://huggingface.co/kandinsky-community/kandinsky-2-2-decoder", "sdxl": "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0", "wuerst": "https://huggingface.co/warp-ai/wuerstchen", "karlo": "https://huggingface.co/kakaobrain/karlo-v1-alpha", } MODEL_KEYS = "-".join(SUBMISSIONS.keys()) SUBMISSION_ORG = f"result-{MODEL_KEYS}" submission_names = list(SUBMISSIONS.keys()) ds = load_dataset("nateraw/parti-prompts")["train"] parti_prompt_categories = ds["Category"] parti_prompt_challenge = ds["Challenge"] def load_submissions(): all_datasets = list_datasets(author=SUBMISSION_ORG) relevant_ids = [d.id for d in all_datasets] ids = defaultdict(list) challenges = defaultdict(list) categories = defaultdict(list) total_submissions = 0 for _id in relevant_ids: try: ds = load_dataset(_id)["train"] except: # skip dataset continue all_results = [] all_ids = [] for result, image_id in zip(ds["result"], ds["id"]): all_result = result.split(",") all_results += all_result all_ids += (len(all_result) * [image_id]) for result, image_id in zip(all_results, all_ids): if result == "": print(f"{result} was not solved by any model.") elif result not in submission_names: import ipdb; ipdb.set_trace() # Make sure that incorrect model names are not added continue ids[result].append(image_id) challenges[parti_prompt_challenge[image_id]].append(result) categories[parti_prompt_categories[image_id]].append(result) total_submissions += 1 all_values = sum(len(v) for v in ids.values()) main_dict = {k: float('{:.2}'.format(len(v)/all_values)) for k, v in ids.items()} challenges = {k: Counter(v) for k, v in challenges.items()} categories = {k: Counter(v) for k, v in categories.items()} return total_submissions, main_dict, challenges, categories def sort_by_highest_percentage(df): # Convert percentage values to numeric format df = df[df.loc[0].sort_values(ascending=False).index] return df def get_dataframe_all(): total_submissions, main, challenges, categories = load_submissions() main_frame = pd.DataFrame([main]) challenges_frame = pd.DataFrame.from_dict(challenges).fillna(0).T challenges_frame = challenges_frame.div(challenges_frame.sum(axis=1), axis=0) categories_frame = pd.DataFrame.from_dict(categories).fillna(0).T categories_frame = categories_frame.div(categories_frame.sum(axis=1), axis=0) main_frame = main_frame.rename(columns={"": "NOT SOLVED"}) categories_frame = categories_frame.rename(columns={"": "NOT SOLVED"}) challenges_frame = challenges_frame.rename(columns={"": "NOT SOLVED"}) main_frame = sort_by_highest_percentage(main_frame) main_frame = main_frame.applymap(lambda x: '{:.2%}'.format(x)) challenges_frame = challenges_frame.applymap(lambda x: '{:.2%}'.format(x)) categories_frame = categories_frame.applymap(lambda x: '{:.2%}'.format(x)) categories_frame = categories_frame.reindex(columns=main_frame.columns.to_list()) challenges_frame = challenges_frame.reindex(columns=main_frame.columns.to_list()) categories_frame = categories_frame.reset_index().rename(columns={'index': 'Category'}) challenges_frame = challenges_frame.reset_index().rename(columns={'index': 'Challenge'}) return total_submissions, main_frame, challenges_frame, categories_frame TITLE = "# Open Parti Prompts Leaderboard" DESCRIPTION = """ The *Open Parti Prompts Leaderboard* compares state-of-the-art, open-source text-to-image models to each other according to **human preferences**. \n\n Text-to-image models are notoriously difficult to evaluate. [FID](https://en.wikipedia.org/wiki/Fr%C3%A9chet_inception_distance) and [CLIP Score](https://en.wikipedia.org/wiki/Fr%C3%A9chet_inception_distance) are not enough to accurately state whether a text-to-image model can **generate "good" images**. "Good" is extremely difficult to put into numbers. \n\n Instead, the **Open Parti Prompts Leaderboard** uses human feedback from the community to compare images from different text-to-image models to each other. \n\n ❤️ ***Please take 3 minutes to contribute to the benchmark.*** \n 👉 ***Play one round of [Open Parti Prompts Game](https://huggingface.co/spaces/OpenGenAI/open-parti-prompts) to contribute 10 answers.*** 🤗 """ EXPLANATION = """\n\n ## How the is data collected 📊 \n\n In more detail, the [Open Parti Prompts Game](https://huggingface.co/spaces/OpenGenAI/open-parti-prompts) collects human preferences that state which generated image best fits a given prompt from the [Parti Prompts](https://huggingface.co/datasets/nateraw/parti-prompts) dataset. Parti Prompts has been designed to challenge text-to-image models on prompts of varying categories and difficulty. The images have been pre-generated from the models that are compared in this space. For more information of how the images were created, please refer to [Open Parti Prompts](https://huggingface.co/spaces/OpenGenAI/open-parti-prompts). The community's answers are then stored and used in this space to give a human-preference-based comparison of the different models. \n\n Currently the leaderboard includes the following models: - [Kandinsky 2.2](https://huggingface.co/kandinsky-community/kandinsky-2-2-decoder) - [Stable Diffusion XL](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) - [Wuerstchen](https://huggingface.co/warp-ai/wuerstchen) - [Karlo](https://huggingface.co/kakaobrain/karlo-v1-alpha) In the following you can see three result tables. The first shows the overall comparison of the 4 models. The score states, **the percentage at which images generated from the corresponding model are preferred over the image from all other models**. The second and third tables show you a breakdown analysis per category and per type of challenge as defined by [Parti Prompts](https://huggingface.co/datasets/nateraw/parti-prompts). """ GALLERY_COLUMN_NUM = len(SUBMISSIONS) def refresh(): return get_dataframe_all() with gr.Blocks() as demo: with gr.Column(visible=True) as intro_view: gr.Markdown(TITLE) gr.Markdown(DESCRIPTION) gr.Markdown(EXPLANATION) headers = list(SUBMISSIONS.keys()) datatype = "str" total_submissions, main_df, challenge_df, category_df = get_dataframe_all() with gr.Column(): gr.Markdown("# Open Parti Prompts") main_dataframe = gr.Dataframe( value=main_df, headers=main_df.columns.to_list(), datatype="str", row_count=main_df.shape[0], col_count=main_df.shape[1], interactive=False, ) with gr.Column(): gr.Markdown("## per category") cat_dataframe = gr.Dataframe( value=category_df, headers=category_df.columns.to_list(), datatype="str", row_count=category_df.shape[0], col_count=category_df.shape[1], interactive=False, ) with gr.Column(): gr.Markdown("## per challenge") chal_dataframe = gr.Dataframe( value=challenge_df, headers=challenge_df.columns.to_list(), datatype="str", row_count=challenge_df.shape[0], col_count=challenge_df.shape[1], interactive=False, ) with gr.Column(): gr.Markdown("## # Submissions") num_submissions = gr.Number(value=total_submissions, interactive=False) with gr.Row(): refresh_button = gr.Button("Refresh") refresh_button.click(refresh, inputs=[], outputs=[num_submissions, main_dataframe, cat_dataframe, chal_dataframe]) # Restart space every 20 minutes scheduler = BackgroundScheduler() scheduler.add_job(restart_space, 'interval', seconds=3600) scheduler.start() demo.launch()