app.py

import os
import re
import copy
import datasets
import pandas as pd
import gradio as gr

from collections import defaultdict
from datetime import datetime, timedelta
from datasets import Dataset
from huggingface_hub import HfApi
from huggingface_hub import create_repo
from huggingface_hub.utils import HfHubHTTPError

import utils
from paper.download import (
    download_pdf_from_arxiv,
    get_papers_from_hf_daily_papers,
    get_papers_from_arxiv_ids
)
from paper.parser import extract_text_and_figures
from gen.gemini import get_basic_qa, get_deep_qa
from constants.styles import STYLE
from constants.js import UPDATE_SEARCH_RESULTS, UPDATE_IF_TYPE

from apscheduler.schedulers.background import BackgroundScheduler

def count_nans(row):
    count = 0

    for _, (k, v) in enumerate(data.items()):
        if v is None:
            count = count + 1

    return count
    
gemini_api_key = os.getenv("GEMINI_API_KEY")
hf_token = os.getenv("HF_TOKEN")
dataset_repo_id = os.getenv("SOURCE_DATA_REPO_ID") # "chansung/auto-paper-qa2"
request_arxiv_repo_id = os.getenv("REQUEST_DATA_REPO_ID") # "chansung/requested-arxiv-ids-3"

ds = datasets.load_dataset(dataset_repo_id)
request_ds = datasets.load_dataset(request_arxiv_repo_id)
requested_arxiv_ids = []
for request_d in request_ds['train']:
    arxiv_ids = request_d['Requested arXiv IDs']
    requested_arxiv_ids = requested_arxiv_ids + arxiv_ids
requested_arxiv_ids_df = pd.DataFrame({'Requested arXiv IDs': requested_arxiv_ids})

title2qna = {}
date2qna = {}
date_dict = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))

for data in ds["train"]:
    date = data["target_date"].strftime("%Y-%m-%d")

    if date in date2qna:
        papers = copy.deepcopy(date2qna[date])
        for paper in papers:
            if paper["title"] == data["title"]:
                if count_nans(paper) > count_nans(data):
                    date2qna[date].remove(paper)
        
        date2qna[date].append(data)
        del papers
    else:
        date2qna[date] = [data]

for date in date2qna:
    year, month, day = date.split("-")
    papers = date2qna[date]
    for paper in papers:
        title2qna[paper["title"]] = paper
        date_dict[year][month][day].append(paper)

titles = title2qna.keys()

sorted_dates = sorted(date2qna.keys())

sorted_year = sorted(date_dict.keys())
last_year = sorted_year[-1]
sorted_month = sorted(date_dict[last_year].keys())
last_month = sorted_month[-1]
sorted_day = sorted(date_dict[last_year][last_month].keys())
last_day = sorted_day[-1]
last_papers = date_dict[last_year][last_month][last_day]
selected_paper = last_papers[0]

def filter_function(example, ids):
    ids_e = example['Requested arXiv IDs']
    for iid in ids:
        if iid in ids_e:
            ids_e.remove(iid)
            example['Requested arXiv IDs'] = ids_e

    print(example)
    return example

def process_arxiv_ids(gemini_api, hf_repo_id, req_hf_repo_id, hf_token, how_many=10):
    arxiv_ids = []

    ds1 = datasets.load_dataset(req_hf_repo_id)
    for d in ds1['train']:
        req_arxiv_ids = d['Requested arXiv IDs']
        if len(req_arxiv_ids) > 0 and req_arxiv_ids[0] != "top":
            arxiv_ids = arxiv_ids + req_arxiv_ids

    arxiv_ids = arxiv_ids[:how_many]

    if arxiv_ids is not None and len(arxiv_ids) > 0:
        print(f"1. Get metadata for the papers [{arxiv_ids}]")
        papers = get_papers_from_arxiv_ids(arxiv_ids)
        print("...DONE")
        
        print("2. Generating QAs for the paper")
        for paper in papers:
            try:
                title = paper['title']
                target_date = paper['target_date']
                abstract = paper['paper']['summary']
                arxiv_id = paper['paper']['id']
                authors = paper['paper']['authors']

                print(f"...PROCESSING ON[{arxiv_id}, {title}]")
                print(f"......Downloading the paper PDF")
                filename = download_pdf_from_arxiv(arxiv_id)
                print(f"......DONE")

                print(f"......Extracting text and figures")
                texts, figures = extract_text_and_figures(filename)
                text =' '.join(texts)
                print(f"......DONE")

                print(f"......Generating the seed(basic) QAs")
                qnas = get_basic_qa(text, gemini_api_key=gemini_api, trucate=30000)
                qnas['title'] = title
                qnas['abstract'] = abstract
                qnas['authors'] = ','.join(authors)
                qnas['arxiv_id'] = arxiv_id
                qnas['target_date'] = target_date
                qnas['full_text'] = text
                print(f"......DONE")

                print(f"......Generating the follow-up QAs")
                qnas = get_deep_qa(text, qnas, gemini_api_key=gemini_api, trucate=30000)
                del qnas["qna"]
                print(f"......DONE")

                print(f"......Exporting to HF Dataset repo at [{hf_repo_id}]")
                utils.push_to_hf_hub(qnas, hf_repo_id, hf_token)
                print(f"......DONE")

                print(f"......Updating request arXiv HF Dataset repo at [{req_hf_repo_id}]")
                ds1 = ds1['train'].map(
                    lambda example: filter_function(example, [arxiv_id])
                ).filter(
                    lambda example: len(example['Requested arXiv IDs']) > 0
                )
                ds1.push_to_hub(req_hf_repo_id, token=hf_token)
                            
                print(f"......DONE")
            except Exception as e:
                print(f".......failed due to exception {e}")
                continue

        HfApi(token=hf_token).restart_space(
            repo_id="chansung/paper_qa", token=hf_token
        )

def push_to_hf_hub(
    df, repo_id, token, append=True
):
    exist = False
    ds = Dataset.from_pandas(df)

    try:
        create_repo(request_arxiv_repo_id, repo_type="dataset", token=hf_token)
    except HfHubHTTPError as e:
        exist = True
        
    if exist and append:
        existing_ds = datasets.load_dataset(repo_id)
        ds = datasets.concatenate_datasets([existing_ds['train'], ds])

    ds.push_to_hub(repo_id, token=token)

def _filter_duplicate_arxiv_ids(arxiv_ids_to_be_added):
    ds1 = datasets.load_dataset("chansung/requested-arxiv-ids-3")
    ds2 = datasets.load_dataset("chansung/auto-paper-qa2")

    unique_arxiv_ids = set()

    for d in ds1['train']:
        arxiv_ids = d['Requested arXiv IDs']
        unique_arxiv_ids = set(list(unique_arxiv_ids) + arxiv_ids)

    for d in ds2['train']:
        arxiv_id = d['arxiv_id']
        unique_arxiv_ids.add(arxiv_id)

    return list(set(arxiv_ids_to_be_added) - unique_arxiv_ids)

def _is_arxiv_id_valid(arxiv_id):
  pattern = r"^\d{4}\.\d{5}$" 
  return bool(re.match(pattern, arxiv_id))

def _get_valid_arxiv_ids(arxiv_ids_str):
    valid_arxiv_ids = []
    invalid_arxiv_ids = []
    
    for arxiv_id in arxiv_ids_str.split(","):
        arxiv_id = arxiv_id.strip()
        if _is_arxiv_id_valid(arxiv_id):
           valid_arxiv_ids.append(arxiv_id)
        else:
            invalid_arxiv_ids.append(arxiv_id)

    return valid_arxiv_ids, invalid_arxiv_ids

def add_arxiv_ids_to_queue(queue, arxiv_ids_str):
    print(0)
    valid_arxiv_ids, invalid_arxiv_ids = _get_valid_arxiv_ids(arxiv_ids_str)
    print("01")
    
    if len(invalid_arxiv_ids) > 0: 
        gr.Warning(f"found invalid arXiv ids as in {invalid_arxiv_ids}")

    if len(valid_arxiv_ids) > 0:
        valid_arxiv_ids = _filter_duplicate_arxiv_ids(valid_arxiv_ids)

        if len(valid_arxiv_ids) > 0:
            valid_arxiv_ids = [[arxiv_id] for arxiv_id in valid_arxiv_ids]
            gr.Warning(f"Processing on [{valid_arxiv_ids}]. Other requested arXiv IDs not found on this list should be already processed or being processed...")
            valid_arxiv_ids = pd.DataFrame({'Requested arXiv IDs': valid_arxiv_ids})
            queue = pd.concat([queue, valid_arxiv_ids])
            queue.reset_index(drop=True)

            push_to_hf_hub(valid_arxiv_ids, request_arxiv_repo_id, hf_token)
        else:
            gr.Warning(f"All requested arXiv IDs are already processed or being processed...")
    else:
        gr.Warning(f"No valid arXiv IDs found...")

    return (
        queue, gr.Textbox("")
    )

def get_paper_by_year(y):
    m = sorted(date_dict[y].keys())
    last_m = m[-1]
    d = sorted(date_dict[y][last_m].keys())
    last_d = d[-1]
    papers = [paper["title"] for paper in date_dict[y][last_m][last_d]]
    papers = list(set(papers))
    return (
        gr.Dropdown(choices=m, value=last_m),
        gr.Dropdown(choices=d, value=last_d),
        gr.Dropdown(choices=papers, value=papers[0])
    )

def get_paper_by_month(y, m):
    d = sorted(date_dict[y][m].keys())
    last_d = d[-1]
    papers = [paper["title"] for paper in date_dict[y][m][last_d]]
    papers = list(set(papers))
    return (
        gr.Dropdown(choices=d, value=last_d),
        gr.Dropdown(choices=papers, value=papers[0])
    )

def get_paper_by_day(y, m, d):
    papers = [paper["title"] for paper in date_dict[y][m][d]]
    papers = list(set(papers))
    return gr.Dropdown(choices=papers, value=papers[0])

def set_paper(y, m, d, paper_title):
    selected_paper = None
    for paper in date_dict[y][m][d]:
        if paper["title"] == paper_title:
            selected_paper = paper
            break

    return (
        gr.Markdown(f"# {selected_paper['title']}"), 
        gr.Markdown(
            "[![arXiv](https://img.shields.io/badge/arXiv-%s-b31b1b.svg)](https://arxiv.org/abs/%s)" % (selected_paper['arxiv_id'], selected_paper['arxiv_id'])
        ),
        gr.Markdown(
            "[![Paper page](https://huggingface.co/datasets/huggingface/badges/resolve/main/paper-page-md.svg)](https://huggingface.co/papers/%s)" % selected_paper['arxiv_id']
        ),
        gr.Markdown(selected_paper["summary"]),

        gr.Markdown(f"### 🙋 {selected_paper['0_question']}"), 
        gr.Markdown(f"↪ **(ELI5)** {selected_paper['0_answers:eli5']}"), 
        gr.Markdown(f"↪ **(Technical)** {selected_paper['0_answers:expert']}"),
        gr.Markdown(f"### 🙋🙋 {selected_paper['0_additional_depth_q:follow up question']}"),
        gr.Markdown(f"↪ **(ELI5)** {selected_paper['0_additional_depth_q:answers:eli5']}"), 
        gr.Markdown(f"↪ **(Technical)** {selected_paper['0_additional_depth_q:answers:expert']}"),
        gr.Markdown(f"### 🙋🙋 {selected_paper['0_additional_breath_q:follow up question']}"), 
        gr.Markdown(f"↪ **(ELI5)** {selected_paper['0_additional_breath_q:answers:eli5']}"), 
        gr.Markdown(f"↪ **(Technical)** {selected_paper['0_additional_breath_q:answers:expert']}"),

        gr.Markdown(f"### 🙋 {selected_paper['1_question']}"), 
        gr.Markdown(f"↪ **(ELI5)** {selected_paper['1_answers:eli5']}"), 
        gr.Markdown(f"↪ **(Technical)** {selected_paper['1_answers:expert']}"),
        gr.Markdown(f"### 🙋🙋 {selected_paper['1_additional_depth_q:follow up question']}"), 
        gr.Markdown(f"↪ **(ELI5)** {selected_paper['1_additional_depth_q:answers:eli5']}"), 
        gr.Markdown(f"↪ **(Technical)** {selected_paper['1_additional_depth_q:answers:expert']}"),
        gr.Markdown(f"### 🙋🙋 {selected_paper['1_additional_breath_q:follow up question']}"), 
        gr.Markdown(f"↪ **(ELI5)** {selected_paper['1_additional_breath_q:answers:eli5']}"), 
        gr.Markdown(f"↪ **(Technical)** {selected_paper['1_additional_breath_q:answers:expert']}"),

        gr.Markdown(f"### 🙋 {selected_paper['2_question']}"), 
        gr.Markdown(f"↪ **(ELI5)** {selected_paper['2_answers:eli5']}"), 
        gr.Markdown(f"↪ **(Technical)** {selected_paper['2_answers:expert']}"),
        gr.Markdown(f"### 🙋🙋 {selected_paper['2_additional_depth_q:follow up question']}"), 
        gr.Markdown(f"↪ **(ELI5)** {selected_paper['2_additional_depth_q:answers:eli5']}"), 
        gr.Markdown(f"↪ **(Technical)** {selected_paper['2_additional_depth_q:answers:expert']}"),
        gr.Markdown(f"### 🙋🙋 {selected_paper['2_additional_breath_q:follow up question']}"), 
        gr.Markdown(f"↪ **(ELI5)** {selected_paper['2_additional_breath_q:answers:eli5']}"), 
        gr.Markdown(f"↪ **(Technical)** {selected_paper['2_additional_breath_q:answers:expert']}"),
    )

def change_exp_type(exp_type):
    if exp_type == "ELI5":
        return (
            gr.Markdown(visible=True), gr.Markdown(visible=False), gr.Markdown(visible=True), gr.Markdown(visible=False), gr.Markdown(visible=True), gr.Markdown(visible=False),
            gr.Markdown(visible=True), gr.Markdown(visible=False), gr.Markdown(visible=True), gr.Markdown(visible=False), gr.Markdown(visible=True), gr.Markdown(visible=False),
            gr.Markdown(visible=True), gr.Markdown(visible=False), gr.Markdown(visible=True), gr.Markdown(visible=False), gr.Markdown(visible=True), gr.Markdown(visible=False),
        )
    else:
        return (
            gr.Markdown(visible=False), gr.Markdown(visible=True), gr.Markdown(visible=False), gr.Markdown(visible=True), gr.Markdown(visible=False), gr.Markdown(visible=True),
            gr.Markdown(visible=False), gr.Markdown(visible=True), gr.Markdown(visible=False), gr.Markdown(visible=True), gr.Markdown(visible=False), gr.Markdown(visible=True),
            gr.Markdown(visible=False), gr.Markdown(visible=True), gr.Markdown(visible=False), gr.Markdown(visible=True), gr.Markdown(visible=False), gr.Markdown(visible=True),
        )        

def search(search_in, max_results=3):
    results = []

    for title in titles:
        if len(results) > 3:
            break
        else:
            if search_in in title:
                results.append(title)

    return (
        gr.Textbox(
            visible=True if len(results) > 0 else False,
            value=results[0] if len(results) > 0 else ""
        ),
        gr.Textbox(
            visible=True if len(results) > 1 else False,
            value=results[1] if len(results) > 1 else ""
        ),
        gr.Textbox(
            visible=True if len(results) > 2 else False,
            value=results[2] if len(results) > 2 else ""
        )
    )

def set_date(title):
    for _, (year, months) in enumerate(date_dict.items()):
        for _, (month, days) in enumerate(months.items()):
            for _, (day, papers) in enumerate(days.items()):
                for paper in papers:
                    if paper['title'] == title:
                        return (
                            gr.Dropdown(value=year),
                            gr.Dropdown(choices=sorted(months), value=month),
                            gr.Dropdown(choices=sorted(days), value=day),
                        )

def set_papers(y, m, d, title):
    papers = [paper["title"] for paper in date_dict[y][m][d]]
    papers = list(set(papers))
    return (
        gr.Dropdown(choices=papers, value=title),
        gr.Textbox("")
    )

with gr.Blocks(css=STYLE, theme=gr.themes.Soft()) as demo:
    gr.Markdown("# Let's explore papers with auto generated Q&As")
    
    with gr.Column(elem_id="control-panel", elem_classes=["group"]):
        with gr.Column():
            with gr.Row():
                year_dd = gr.Dropdown(sorted_year, value=last_year, label="Year", interactive=True, filterable=False)    
                month_dd = gr.Dropdown(sorted_month, value=last_month, label="Month", interactive=True, filterable=False)    
                day_dd = gr.Dropdown(sorted_day, value=last_day, label="Day", interactive=True, filterable=False)
                
            papers_dd = gr.Dropdown(
                list(set([paper["title"] for paper in last_papers])),
                value=selected_paper["title"],
                label="Select paper title", 
                interactive=True,
                filterable=False
            )

        with gr.Column(elem_classes=["no-gap"]):
            search_in = gr.Textbox("", placeholder="Enter keywords to search...", elem_classes=["textbox-no-label"])
            search_r1 = gr.Button(visible=False, elem_id="search_r1", elem_classes=["no-radius"])
            search_r2 = gr.Button(visible=False, elem_id="search_r2", elem_classes=["no-radius"])
            search_r3 = gr.Button(visible=False, elem_id="search_r3", elem_classes=["no-radius"])
            search_r4 = gr.Button(visible=False, elem_id="search_r4", elem_classes=["no-radius"])
            search_r5 = gr.Button(visible=False, elem_id="search_r5", elem_classes=["no-radius"])
            search_r6 = gr.Button(visible=False, elem_id="search_r6", elem_classes=["no-radius"])
            search_r7 = gr.Button(visible=False, elem_id="search_r7", elem_classes=["no-radius"])
            search_r8 = gr.Button(visible=False, elem_id="search_r8", elem_classes=["no-radius"])
            search_r9 = gr.Button(visible=False, elem_id="search_r9", elem_classes=["no-radius"])
            search_r10 = gr.Button(visible=False, elem_id="search_r10", elem_classes=["no-radius"])

        conv_type = gr.Radio(choices=["Q&As", "Chat"], value="Q&As", interactive=True, visible=False, elem_classes=["conv-type"])

    with gr.Column(scale=7):
        title = gr.Markdown(f"# {selected_paper['title']}")
        # with gr.Row():
        with gr.Row():
            arxiv_link = gr.Markdown(
                "[![arXiv](https://img.shields.io/badge/arXiv-%s-b31b1b.svg)](https://arxiv.org/abs/%s)" % (selected_paper['arxiv_id'], selected_paper['arxiv_id'])
            )
            hf_paper_link = gr.Markdown(
                "[![Paper page](https://huggingface.co/datasets/huggingface/badges/resolve/main/paper-page-md.svg)](https://huggingface.co/papers/%s)" % selected_paper['arxiv_id']
            )
            gr.Button("Chat about the paper", interactive=False)
            
        summary = gr.Markdown(f"{selected_paper['summary']}", elem_classes=["small-font"])

        with gr.Column(elem_id="chat_block", visible=False):
            gr.Chatbot([("hello", "world"), ("how", "are you?")])

        with gr.Column(elem_id="qna_block", visible=True):
            with gr.Row():
                with gr.Column(scale=7):
                    gr.Markdown("## Auto generated Questions & Answers")

                exp_type = gr.Radio(choices=["ELI5", "Technical"], value="ELI5", elem_classes=["exp-type"], scale=3)

            # 1
            with gr.Column(elem_classes=["group"], visible=True) as q_0:
                basic_q_0 = gr.Markdown(f"### 🙋 {selected_paper['0_question']}")
                basic_q_eli5_0 = gr.Markdown(f"↪ **(ELI5)** {selected_paper['0_answers:eli5']}", elem_classes=["small-font"]) 
                basic_q_expert_0 = gr.Markdown(f"↪ **(Technical)** {selected_paper['0_answers:expert']}", visible=False, elem_classes=["small-font"]) 

                with gr.Accordion("Additional question #1", open=False, elem_classes=["accordion"]) as aq_0_0:
                    depth_q_0 = gr.Markdown(f"### 🙋🙋 {selected_paper['0_additional_depth_q:follow up question']}")
                    depth_q_eli5_0 = gr.Markdown(f"↪ **(ELI5)** {selected_paper['0_additional_depth_q:answers:eli5']}", elem_classes=["small-font"])
                    depth_q_expert_0 = gr.Markdown(f"↪ **(Technical)** {selected_paper['0_additional_depth_q:answers:expert']}", visible=False, elem_classes=["small-font"])

                with gr.Accordion("Additional question #2", open=False, elem_classes=["accordion"]) as aq_0_1:
                    breath_q_0 = gr.Markdown(f"### 🙋🙋 {selected_paper['0_additional_breath_q:follow up question']}")
                    breath_q_eli5_0 = gr.Markdown(f"↪ **(ELI5)** {selected_paper['0_additional_breath_q:answers:eli5']}", elem_classes=["small-font"])
                    breath_q_expert_0 = gr.Markdown(f"↪ **(Technical)** {selected_paper['0_additional_breath_q:answers:expert']}", visible=False, elem_classes=["small-font"])

            # 2
            with gr.Column(elem_classes=["group"], visible=True) as q_1:
                basic_q_1 = gr.Markdown(f"### 🙋 {selected_paper['1_question']}")
                basic_q_eli5_1 = gr.Markdown(f"↪ **(ELI5)** {selected_paper['1_answers:eli5']}", elem_classes=["small-font"]) 
                basic_q_expert_1 = gr.Markdown(f"↪ **(Technical)** {selected_paper['1_answers:expert']}", visible=False, elem_classes=["small-font"]) 

                with gr.Accordion("Additional question #1", open=False, elem_classes=["accordion"]) as aq_1_0:
                    depth_q_1 = gr.Markdown(f"### 🙋🙋 {selected_paper['1_additional_depth_q:follow up question']}")
                    depth_q_eli5_1 = gr.Markdown(f"↪ **(ELI5)** {selected_paper['1_additional_depth_q:answers:eli5']}", elem_classes=["small-font"])
                    depth_q_expert_1 = gr.Markdown(f"↪ **(Technical)** {selected_paper['1_additional_depth_q:answers:expert']}", visible=False, elem_classes=["small-font"])

                with gr.Accordion("Additional question #2", open=False, elem_classes=["accordion"]) as aq_1_1:
                    breath_q_1 = gr.Markdown(f"### 🙋🙋 {selected_paper['1_additional_breath_q:follow up question']}")
                    breath_q_eli5_1 = gr.Markdown(f"↪ **(ELI5)** {selected_paper['1_additional_breath_q:answers:eli5']}", elem_classes=["small-font"])
                    breath_q_expert_1 = gr.Markdown(f"↪ **(Technical)** {selected_paper['1_additional_breath_q:answers:expert']}", visible=False, elem_classes=["small-font"])

            # 3
            with gr.Column(elem_classes=["group"], visible=True) as q_2:
                basic_q_2 = gr.Markdown(f"### 🙋 {selected_paper['2_question']}")
                basic_q_eli5_2 = gr.Markdown(f"↪ **(ELI5)** {selected_paper['2_answers:eli5']}", elem_classes=["small-font"]) 
                basic_q_expert_2 = gr.Markdown(f"↪ **(Technical)** {selected_paper['2_answers:expert']}", visible=False, elem_classes=["small-font"]) 

                with gr.Accordion("Additional question #1", open=False, elem_classes=["accordion"]) as aq_2_0:
                    depth_q_2 = gr.Markdown(f"### 🙋🙋 {selected_paper['2_additional_depth_q:follow up question']}")
                    depth_q_eli5_2 = gr.Markdown(f"↪ **(ELI5)** {selected_paper['2_additional_depth_q:answers:eli5']}", elem_classes=["small-font"])
                    depth_q_expert_2 = gr.Markdown(f"↪ **(Technical)** {selected_paper['2_additional_depth_q:answers:expert']}", visible=False, elem_classes=["small-font"])

                with gr.Accordion("Additional question #2", open=False, elem_classes=["accordion"]) as aq_2_1:
                    breath_q_2 = gr.Markdown(f"### 🙋🙋 {selected_paper['2_additional_breath_q:follow up question']}")
                    breath_q_eli5_2 = gr.Markdown(f"↪ **(ELI5)** {selected_paper['2_additional_breath_q:answers:eli5']}", elem_classes=["small-font"])
                    breath_q_expert_2 = gr.Markdown(f"↪ **(Technical)** {selected_paper['2_additional_breath_q:answers:expert']}", visible=False, elem_classes=["small-font"])

        gr.Markdown("## Request any arXiv ids")
        arxiv_queue = gr.Dataframe(
            headers=["Requested arXiv IDs"], col_count=(1, "fixed"),
            value=requested_arxiv_ids_df,
            datatype=["str"],
            interactive=False
        )

        arxiv_id_enter = gr.Textbox(placeholder="Enter comma separated arXiv IDs...", elem_classes=["textbox-no-label"])
        arxiv_id_enter.submit(
            add_arxiv_ids_to_queue,
            [arxiv_queue, arxiv_id_enter],
            [arxiv_queue, arxiv_id_enter]
        )


    gr.Markdown("The target papers are collected from [Hugging Face 🤗 Daily Papers](https://huggingface.co/papers) on a daily basis. "
                "The entire data is generated by [Google's Gemini 1.0](https://deepmind.google/technologies/gemini/) Pro. "
                "If you are curious how it is done, visit the [Auto Paper Q&A Generation project repository](https://github.com/deep-diver/auto-paper-analysis) "
                "Also, the generated dataset is hosted on Hugging Face 🤗 Dataset repository as well([Link](https://huggingface.co/datasets/chansung/auto-paper-qa2)). ")
    
    search_r1.click(set_date, search_r1, [year_dd, month_dd, day_dd]).then(
        set_papers,
        inputs=[year_dd, month_dd, day_dd, search_r1],
        outputs=[papers_dd, search_in]
    )

    search_r2.click(set_date, search_r2, [year_dd, month_dd, day_dd]).then(
        set_papers,
        inputs=[year_dd, month_dd, day_dd, search_r2],
        outputs=[papers_dd, search_in]
    )

    search_r3.click(set_date, search_r3, [year_dd, month_dd, day_dd]).then(
        set_papers,
        inputs=[year_dd, month_dd, day_dd, search_r3],
        outputs=[papers_dd, search_in]
    )

    search_r4.click(set_date, search_r4, [year_dd, month_dd, day_dd]).then(
        set_papers,
        inputs=[year_dd, month_dd, day_dd, search_r4],
        outputs=[papers_dd, search_in]
    )

    search_r5.click(set_date, search_r5, [year_dd, month_dd, day_dd]).then(
        set_papers,
        inputs=[year_dd, month_dd, day_dd, search_r5],
        outputs=[papers_dd, search_in]
    )

    search_r6.click(set_date, search_r6, [year_dd, month_dd, day_dd]).then(
        set_papers,
        inputs=[year_dd, month_dd, day_dd, search_r6],
        outputs=[papers_dd, search_in]
    )

    search_r7.click(set_date, search_r7, [year_dd, month_dd, day_dd]).then(
        set_papers,
        inputs=[year_dd, month_dd, day_dd, search_r7],
        outputs=[papers_dd, search_in]
    )    

    search_r8.click(set_date, search_r8, [year_dd, month_dd, day_dd]).then(
        set_papers,
        inputs=[year_dd, month_dd, day_dd, search_r8],
        outputs=[papers_dd, search_in]
    )

    search_r9.click(set_date, search_r9, [year_dd, month_dd, day_dd]).then(
        set_papers,
        inputs=[year_dd, month_dd, day_dd, search_r9],
        outputs=[papers_dd, search_in]
    )

    search_r10.click(set_date, search_r10, [year_dd, month_dd, day_dd]).then(
        set_papers,
        inputs=[year_dd, month_dd, day_dd, search_r10],
        outputs=[papers_dd, search_in]
    )

    year_dd.input(
        get_paper_by_year, 
        inputs=[year_dd],
        outputs=[month_dd, day_dd, papers_dd]
    ).then(
        set_paper,
        [year_dd, month_dd, day_dd, papers_dd],
        [
            title, summary,
            basic_q_0, basic_q_eli5_0, basic_q_expert_0,
            depth_q_0, depth_q_eli5_0, depth_q_expert_0,
            breath_q_0, breath_q_eli5_0, breath_q_expert_0,

            basic_q_1, basic_q_eli5_1, basic_q_expert_1,
            depth_q_1, depth_q_eli5_1, depth_q_expert_1,
            breath_q_1, breath_q_eli5_1, breath_q_expert_1,

            basic_q_2, basic_q_eli5_2, basic_q_expert_2,
            depth_q_2, depth_q_eli5_2, depth_q_expert_2,
            breath_q_2, breath_q_eli5_2, breath_q_expert_2
        ]        
    )

    month_dd.input(
        get_paper_by_month, 
        inputs=[year_dd, month_dd], 
        outputs=[day_dd, papers_dd]
    ).then(
        set_paper,
        [year_dd, month_dd, day_dd, papers_dd],
        [
            title, arxiv_link, hf_paper_link, summary,
            basic_q_0, basic_q_eli5_0, basic_q_expert_0,
            depth_q_0, depth_q_eli5_0, depth_q_expert_0,
            breath_q_0, breath_q_eli5_0, breath_q_expert_0,

            basic_q_1, basic_q_eli5_1, basic_q_expert_1,
            depth_q_1, depth_q_eli5_1, depth_q_expert_1,
            breath_q_1, breath_q_eli5_1, breath_q_expert_1,

            basic_q_2, basic_q_eli5_2, basic_q_expert_2,
            depth_q_2, depth_q_eli5_2, depth_q_expert_2,
            breath_q_2, breath_q_eli5_2, breath_q_expert_2
        ]        
    )

    day_dd.input(
        get_paper_by_day, 
        inputs=[year_dd, month_dd, day_dd], 
        outputs=[papers_dd]
    ).then(
        set_paper,
        [year_dd, month_dd, day_dd, papers_dd],
        [
            title, arxiv_link, hf_paper_link, summary,
            basic_q_0, basic_q_eli5_0, basic_q_expert_0,
            depth_q_0, depth_q_eli5_0, depth_q_expert_0,
            breath_q_0, breath_q_eli5_0, breath_q_expert_0,

            basic_q_1, basic_q_eli5_1, basic_q_expert_1,
            depth_q_1, depth_q_eli5_1, depth_q_expert_1,
            breath_q_1, breath_q_eli5_1, breath_q_expert_1,

            basic_q_2, basic_q_eli5_2, basic_q_expert_2,
            depth_q_2, depth_q_eli5_2, depth_q_expert_2,
            breath_q_2, breath_q_eli5_2, breath_q_expert_2
        ]        
    )

    papers_dd.change(
        set_paper,
        [year_dd, month_dd, day_dd, papers_dd],
        [
            title, arxiv_link, hf_paper_link, summary,
            basic_q_0, basic_q_eli5_0, basic_q_expert_0,
            depth_q_0, depth_q_eli5_0, depth_q_expert_0,
            breath_q_0, breath_q_eli5_0, breath_q_expert_0,

            basic_q_1, basic_q_eli5_1, basic_q_expert_1,
            depth_q_1, depth_q_eli5_1, depth_q_expert_1,
            breath_q_1, breath_q_eli5_1, breath_q_expert_1,

            basic_q_2, basic_q_eli5_2, basic_q_expert_2,
            depth_q_2, depth_q_eli5_2, depth_q_expert_2,
            breath_q_2, breath_q_eli5_2, breath_q_expert_2
        ]
    )

    search_in.change(
        inputs=[search_in],
        outputs=[
            search_r1, search_r2, search_r3, search_r4, search_r5,
            search_r6, search_r7, search_r8, search_r9, search_r10
        ],
        js=UPDATE_SEARCH_RESULTS % str(list(titles)),
        fn=None
    )

    exp_type.select(
        change_exp_type,
        exp_type,
        [
            basic_q_eli5_0, basic_q_expert_0, depth_q_eli5_0, depth_q_expert_0, breath_q_eli5_0, breath_q_expert_0,
            basic_q_eli5_1, basic_q_expert_1, depth_q_eli5_1, depth_q_expert_1, breath_q_eli5_1, breath_q_expert_1,
            basic_q_eli5_2, basic_q_expert_2, depth_q_eli5_2, depth_q_expert_2, breath_q_eli5_2, breath_q_expert_2
        ]
    )

    conv_type.select(
        inputs=[conv_type],
        js=UPDATE_IF_TYPE,
        outputs=None,
        fn=None
    )

start_date = datetime.now() + timedelta(minutes=1)
scheduler = BackgroundScheduler()
scheduler.add_job(
    process_arxiv_ids,
    trigger='interval',
    seconds=3600,
    args=[
        gemini_api_key, 
        dataset_repo_id,
        request_arxiv_repo_id,
        hf_token
    ],
    start_date=start_date
)
scheduler.start()

demo.launch(share=True, debug=True)