app.py

import gradio as gr
import pandas as pd
import os
from apscheduler.schedulers.background import BackgroundScheduler
from huggingface_hub import HfApi
from uploads import add_new_eval

CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
CITATION_BUTTON_TEXT = r"""@inproceedings{iltur-2024,
      title = "IL-TUR: Benchmark for Indian Legal Text Understanding and Reasoning",
      author = "Joshi, Abhinav and Paul, Shounak and Sharma, Akshat and Goyal, Pawan and Ghosh, Saptarshi and Modi, Ashutosh"
      booktitle = "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
      month = aug,
      year = "2024",
      address = "Bangkok, Thailand",
      publisher = "Association for Computational Linguistics",
  }"""

api = HfApi()
TOKEN = os.environ.get("TOKEN", None)
LEADERBOARD_PATH = f"Exploration-lab/IL-TUR-Leaderboard"


def restart_space():
    api.restart_space(repo_id=LEADERBOARD_PATH, token=TOKEN)


# Function to load data from a given CSV file
def baseline_load_data(tasks, task_metrics):

    import json

    # load the results json file
    with open("submissions/baseline/results.json") as f:
        results = json.load(f)

    # create a new df to display the results
    results_df = pd.DataFrame(
        columns=[
            "Method",
            "Submitted By",
            "Github Link",
            "L-NER",
            "RR",
            "CJPE",
            "BAIL",
            "LSI",
            "PCR",
            "SUMM",
            "L-MT",
            # "Average",
        ]
    )
    # breakpoint()
    for entry in results:
        results_df = results_df.append(
            {
                "Method": entry["Method"],
                "Submitted By": entry["Submitted By"],
                "Github Link": entry["Github Link"],
                "L-NER": entry["L-NER"][task_metrics["L-NER"]],
                "RR": entry["RR"][task_metrics["RR"]],
                "CJPE": entry["CJPE"][task_metrics["CJPE"]],
                "BAIL": entry["BAIL"][task_metrics["BAIL"]],
                "LSI": entry["LSI"][task_metrics["LSI"]],
                "PCR": entry["PCR"][task_metrics["PCR"]],
                "SUMM": entry["SUMM"][task_metrics["SUMM"]],
                "L-MT": entry["L-MT"][task_metrics["L-MT"]],
                # "Average": ,
            },
            ignore_index=True,
        )

    df = results_df
    # remove the columns that are not in tasks
    selected_columns = (
        [
            "Method",
            "Submitted By",
        ]
        + tasks
        + ["Github Link"]
    )
    df = df[selected_columns]

    df = df.drop_duplicates(subset=["Method"], keep="first")

    return df


def load_data(tasks, task_metrics):
    baseline_df = baseline_load_data(tasks, task_metrics)

    return baseline_df


# Function for searching in the leaderboard
def search_leaderboard(df, query):
    if query == "":
        return df
    else:
        return df[df["Method"].str.contains(query)]


# Function to change the version of the leaderboard
def change_version(
    tasks,
    l_ner_metric,
    rr_metric,
    cjpe_metric,
    bail_metric,
    lsi_metric,
    pcr_metric,
    summ_metric,
    lmt_metric,
):
    task_metrics = {
        "L-NER": l_ner_metric,
        "RR": rr_metric,
        "CJPE": cjpe_metric,
        "BAIL": bail_metric,
        "LSI": lsi_metric,
        "PCR": pcr_metric,
        "SUMM": summ_metric,
        "L-MT": lmt_metric,
    }

    new_df = load_data(tasks, task_metrics)
    return new_df


# Initialize Gradio app
demo = gr.Blocks()

with demo:
    gr.Markdown(
        """
    ## 🥇 IL-TUR Leaderboard
    Legal systems worldwide are inundated with exponential growth in cases and documents. There is an imminent need to develop NLP and ML techniques for automatically processing and understanding legal documents to streamline the legal system. However, evaluating and comparing various NLP models designed specifically for the legal domain is challenging. This paper addresses this challenge by proposing IL-TUR: Benchmark for Indian Legal Text Understanding and Reasoning. IL-TUR  contains monolingual (English, Hindi) and multi-lingual (9 Indian languages) domain-specific tasks that address different aspects of the legal system from the point of view of understanding and reasoning over Indian legal documents. We present baseline models (including LLM-based) for each task, outlining the gap between models and the ground truth. We will release a public leaderboard where the research community can upload and compare legal text understanding systems on various metrics, thus fostering research in the legal domain.
    Read more at [https://exploration-lab.github.io/IL-TUR/](https://exploration-lab.github.io/IL-TUR/).
    """
    )

    with gr.Row():
        with gr.Accordion("📙 Citation", open=False):
            citation_button = gr.Textbox(
                value=CITATION_BUTTON_TEXT,
                label=CITATION_BUTTON_LABEL,
                elem_id="citation-button",
                show_copy_button=True,
            )  # .style(show_copy_button=True)

    with gr.Tabs():
        with gr.TabItem("Leaderboard"):

            with gr.Row():
                tasks_checkbox = gr.CheckboxGroup(
                    label="Select Tasks",
                    choices=[
                        "L-NER",
                        "RR",
                        "CJPE",
                        "BAIL",
                        "LSI",
                        "PCR",
                        "SUMM",
                        "L-MT",
                    ],
                    value=["L-NER", "RR", "CJPE", "BAIL", "LSI", "PCR", "SUMM", "L-MT"],
                    interactive=True,
                )

            with gr.Row():
                l_ner_metric = gr.Radio(
                    label="L-NER",
                    choices=["strict mF1"],
                    value="strict mF1",
                    interactive=True,
                )
                rr_metric = gr.Radio(
                    label="RR",
                    choices=["mF1"],
                    value="mF1",
                    interactive=True,
                )
                cjpe_metric = gr.Radio(
                    label="CJPE",
                    choices=["mF1", "ROUGE-L", "BLEU"],
                    value="mF1",
                    interactive=True,
                )
                bail_metric = gr.Radio(
                    label="BAIL",
                    choices=["mF1"],
                    value="mF1",
                    interactive=True,
                )
                lsi_metric = gr.Radio(
                    label="LSI",
                    choices=["mF1"],
                    value="mF1",
                    interactive=True,
                )
                pcr_metric = gr.Radio(
                    label="PCR",
                    choices=["muF1@K"],
                    value="muF1@K",
                    interactive=True,
                )
                summ_metric = gr.Radio(
                    label="SUMM",
                    choices=["ROUGE-L", "BERTSCORE"],
                    value="ROUGE-L",
                    interactive=True,
                )
                lmt_metric = gr.Radio(
                    label="L-MT",
                    choices=["BLEU", "GLEU", "chrF++"],
                    value="BLEU",
                    interactive=True,
                )

            with gr.Row():
                search_bar = gr.Textbox(
                    placeholder="Search for methods...",
                    show_label=False,
                )

            task_metrics = {
                "L-NER": l_ner_metric.value,
                "RR": rr_metric.value,
                "CJPE": cjpe_metric.value,
                "BAIL": bail_metric.value,
                "LSI": lsi_metric.value,
                "PCR": pcr_metric.value,
                "SUMM": summ_metric.value,
                "L-MT": lmt_metric.value,
            }

            leaderboard_table = gr.components.Dataframe(
                value=load_data(
                    # "baseline",
                    ["L-NER", "RR", "CJPE", "BAIL", "LSI", "PCR", "SUMM", "L-MT"],
                    task_metrics=task_metrics,
                ),
                interactive=True,
                visible=True,
            )

            # version_dropdown.change(
            #     change_version,
            #     inputs=[model_dropdown, version_dropdown, tasks_checkbox],
            #     outputs=leaderboard_table,
            # )

            # model_dropdown.change(
            #     change_version,
            #     inputs=[model_dropdown, version_dropdown, tasks_checkbox],
            #     outputs=leaderboard_table,
            # )

            search_bar.change(
                search_leaderboard,
                inputs=[leaderboard_table, search_bar],
                outputs=leaderboard_table,
            )
            # breakpoint()
            l_ner_metric.change(
                change_version,
                inputs=[
                    tasks_checkbox,
                    l_ner_metric,
                    rr_metric,
                    cjpe_metric,
                    bail_metric,
                    lsi_metric,
                    pcr_metric,
                    summ_metric,
                    lmt_metric,
                ],
                outputs=leaderboard_table,
            )
            rr_metric.change(
                change_version,
                inputs=[
                    tasks_checkbox,
                    l_ner_metric,
                    rr_metric,
                    cjpe_metric,
                    bail_metric,
                    lsi_metric,
                    pcr_metric,
                    summ_metric,
                    lmt_metric,
                ],
                outputs=leaderboard_table,
            )
            cjpe_metric.change(
                change_version,
                inputs=[
                    tasks_checkbox,
                    l_ner_metric,
                    rr_metric,
                    cjpe_metric,
                    bail_metric,
                    lsi_metric,
                    pcr_metric,
                    summ_metric,
                    lmt_metric,
                ],
                outputs=leaderboard_table,
            )
            bail_metric.change(
                change_version,
                inputs=[
                    tasks_checkbox,
                    l_ner_metric,
                    rr_metric,
                    cjpe_metric,
                    bail_metric,
                    lsi_metric,
                    pcr_metric,
                    summ_metric,
                    lmt_metric,
                ],
                outputs=leaderboard_table,
            )
            lsi_metric.change(
                change_version,
                inputs=[
                    tasks_checkbox,
                    l_ner_metric,
                    rr_metric,
                    cjpe_metric,
                    bail_metric,
                    lsi_metric,
                    pcr_metric,
                    summ_metric,
                    lmt_metric,
                ],
                outputs=leaderboard_table,
            )
            pcr_metric.change(
                change_version,
                inputs=[
                    tasks_checkbox,
                    l_ner_metric,
                    rr_metric,
                    cjpe_metric,
                    bail_metric,
                    lsi_metric,
                    pcr_metric,
                    summ_metric,
                    lmt_metric,
                ],
                outputs=leaderboard_table,
            )
            summ_metric.change(
                change_version,
                inputs=[
                    tasks_checkbox,
                    l_ner_metric,
                    rr_metric,
                    cjpe_metric,
                    bail_metric,
                    lsi_metric,
                    pcr_metric,
                    summ_metric,
                    lmt_metric,
                ],
                outputs=leaderboard_table,
            )
            lmt_metric.change(
                change_version,
                inputs=[
                    tasks_checkbox,
                    l_ner_metric,
                    rr_metric,
                    cjpe_metric,
                    bail_metric,
                    lsi_metric,
                    pcr_metric,
                    summ_metric,
                    lmt_metric,
                ],
                outputs=leaderboard_table,
            )
            tasks_checkbox.change(
                change_version,
                inputs=[
                    tasks_checkbox,
                    l_ner_metric,
                    rr_metric,
                    cjpe_metric,
                    bail_metric,
                    lsi_metric,
                    pcr_metric,
                    summ_metric,
                    lmt_metric,
                ],
                outputs=leaderboard_table,
            )

    with gr.Accordion("Submit the Results of your Method"):
        with gr.Row():
            with gr.Column():
                method_name_textbox = gr.Textbox(
                    label="Method",
                )
                submitted_by_textbox = gr.Textbox(label="Submitted By (Team Name)")
                url_textbox = gr.Textbox(label="Github Link")
                organisation = gr.Textbox(label="Organisation")
                mail = gr.Textbox(label="Contact email")
            with gr.Column():
                file_output = gr.File()
                submit_button = gr.Button("Submit Eval")
        submission_result = gr.Markdown()
        submit_button.click(
            add_new_eval,
            [
                method_name_textbox,
                submitted_by_textbox,
                url_textbox,
                file_output,
                organisation,
                mail,
            ],
            submission_result,
        )

    gr.Markdown(
        """
    ## Quick Links

    - [**Website**](https://exploration-lab.github.io/IL-TUR): The landing page for IL-TUR
    - [**arXiv Paper**](https://arxiv.org/abs/2307.05260): Detailed information about the IL-TUR dataset and its significance in unlearning tasks.
    - [**GitHub Repository**](https://github.com/exploration-lab/IL-TUR): Access the source code, fine-tuning scripts, and additional resources for the IL-TUR dataset.
    - [**Dataset on Hugging Face**](https://huggingface.co/datasets/Exploration-Lab/IL-TUR): Direct link to download the IL-TUR dataset.
    - [**Leaderboard on Hugging Face Spaces**](https://huggingface.co/spaces/Exploration-Lab/IL-TUR_leaderboard): Current rankings and submissions for the IL-TUR dataset challenges.
    
    ## Loading the Dataset
    
    To load the dataset, use the following code:
    
    ```python
    from datasets import load_dataset
    dataset = load_dataset("Exploration-Lab/IL-TUR", "<task_name>", revision="script")
    ```

    ## Creating a submission file
    
    A submission file should exactly follow the format as ["IL_TUR_eval_submission_dummy.json"](https://huggingface.co/spaces/Exploration-Lab/IL-TUR-Leaderboard/blob/main/submissions/baseline/IL_TUR_eval_submission_dummy.json).
    Each key in the file corresponds to each task. You can submit predictions for one, multiple, or all tasks.
    However, for any task you submit, you should have predictions corresponding to every instance in the test set (keys in the submission file).
    In most cases, the format of the predictions is similar to that of the gold-standard labels in the dataset.
    """
    )

# scheduler = BackgroundScheduler()
# scheduler.add_job(restart_space, "interval", seconds=1800)
# scheduler.start()
# demo.queue(default_concurrency_limit=40).launch()

# demo.launch()
scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", seconds=3600)
scheduler.start()
# demo.launch(debug=True)
demo.launch(share=True)