Dataframe now updates on background (#5)

- Dataframe now updates on background (ec79be5278aa2322d671aabf69504a90938fde6d)

app.py

@@ -1,3 +1,5 @@
+from apscheduler.schedulers.background import BackgroundScheduler
+import datetime
 import os
 from typing import Dict, Tuple
 from uuid import UUID
@@ -214,6 +216,7 @@ def kpi_chart() -> alt.Chart:
 
     return chart
 
+
 def render_hub_user_link(hub_id):
     link = f"https://huggingface.co/{hub_id}"
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{hub_id}</a>'
@@ -233,13 +236,42 @@ def obtain_top_5_users(user_ids_annotations: Dict[str, int]) -> pd.DataFrame:
     dataframe = pd.DataFrame(
         user_ids_annotations.items(), columns=["Name", "Submitted Responses"]
     )
-    dataframe['Name'] = dataframe['Name'].apply(render_hub_user_link)
+    dataframe["Name"] = dataframe["Name"].apply(render_hub_user_link)
     dataframe = dataframe.sort_values(by="Submitted Responses", ascending=False)
     return dataframe.head(50)
 
 
+def fetch_data() -> None:
+    """
+    This function fetches the data from the source and target datasets and updates the global variables.
+    """
+
+    print(f"Starting to fetch data: {datetime.datetime.now()}")
+
+    global source_dataset, target_dataset, user_ids_annotations, annotated, remaining, percentage_completed, top5_dataframe
+    source_dataset, target_dataset = obtain_source_target_datasets()
+    user_ids_annotations = get_user_annotations_dictionary(target_dataset)
+
+    annotated = len(target_dataset)
+    remaining = int(os.getenv("TARGET_RECORDS")) - annotated
+    percentage_completed = round(
+        (annotated / int(os.getenv("TARGET_RECORDS"))) * 100, 1
+    )
+
+    # Print the current date and time
+    print(f"Data fetched: {datetime.datetime.now()}")
+
+
+def get_top5() -> pd.DataFrame:
+    return obtain_top_5_users(user_ids_annotations)
+
+
 def main() -> None:
 
+    # Set the update interval
+    update_interval = 300  # seconds
+    update_interval_charts = 30  # seconds
+
     # Connect to the space with rg.init()
     rg.init(
         api_url=os.getenv("ARGILLA_API_URL"),
@@ -247,19 +279,22 @@ def main() -> None:
         extra_headers={"Authorization": f"Bearer {os.getenv('HF_TOKEN')}"},
     )
 
-    global source_dataset, target_dataset, user_ids_annotations
-    source_dataset, target_dataset = obtain_source_target_datasets()
-    user_ids_annotations = get_user_annotations_dictionary(target_dataset)
+    fetch_data()
 
-    top5_dataframe = obtain_top_5_users(user_ids_annotations)
-
-    annotated = len(target_dataset)
-    remaining = int(os.getenv("TARGET_RECORDS")) - annotated
-    percentage_completed = round(
-        (annotated / int(os.getenv("TARGET_RECORDS"))) * 100, 1
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(
+        func=fetch_data, trigger="interval", seconds=update_interval, max_instances=1
     )
+    scheduler.start()
 
-    with gr.Blocks() as demo:
+    # To avoid the orange border for the Gradio elements that are in constant loading
+    css = """
+    .generating {
+        border: none;
+    }
+    """
+
+    with gr.Blocks(css=css) as demo:
         gr.Markdown(
             """
             # 🗣️ The Prompt Collective Dashboad
@@ -277,11 +312,12 @@ def main() -> None:
         )
         with gr.Row():
 
-            plot2 = gr.Plot(label="Plot")
+            donut_target_plot = gr.Plot(label="Plot")
             demo.load(
                 donut_chart_target,
                 inputs=[],
-                outputs=[plot2],
+                outputs=[donut_target_plot],
+                every=update_interval_charts,
             )
 
         gr.Markdown(
@@ -299,25 +335,28 @@ def main() -> None:
         )
         with gr.Row():
 
-            plot = gr.Plot(label="Plot")
+            kpi_submitted_plot = gr.Plot(label="Plot")
             demo.load(
                 kpi_chart_submitted,
                 inputs=[],
-                outputs=[plot],
+                outputs=[kpi_submitted_plot],
+                every=update_interval_charts,
             )
 
-            plot = gr.Plot(label="Plot")
+            kpi_remaining_plot = gr.Plot(label="Plot")
             demo.load(
                 kpi_chart_remaining,
                 inputs=[],
-                outputs=[plot],
+                outputs=[kpi_remaining_plot],
+                every=update_interval_charts,
             )
 
-            plot2 = gr.Plot(label="Plot")
+            donut_total_plot = gr.Plot(label="Plot")
             demo.load(
                 donut_chart_total,
                 inputs=[],
-                outputs=[plot2],
+                outputs=[donut_total_plot],
+                every=update_interval_charts,
             )
 
         gr.Markdown(
@@ -329,15 +368,12 @@ def main() -> None:
 
         with gr.Row():
 
-            plot2 = gr.Plot(label="Plot")
+            kpi_hall_plot = gr.Plot(label="Plot")
             demo.load(
-                kpi_chart,
-                inputs=[],
-                outputs=[plot2],
+                kpi_chart, inputs=[], outputs=[kpi_hall_plot], every=update_interval_charts
             )
 
-            gr.Dataframe(
-                value=top5_dataframe,
+            top5_df_plot = gr.Dataframe(
                 headers=["Name", "Submitted Responses"],
                 datatype=[
                     "markdown",
@@ -346,7 +382,9 @@ def main() -> None:
                 row_count=50,
                 col_count=(2, "fixed"),
                 interactive=False,
-            ),
+                every=update_interval,
+            )
+            demo.load(get_top5, None, [top5_df_plot], every=update_interval_charts)
 
     # Launch the Gradio interface
     demo.launch()

requirements.txt

@@ -2,6 +2,7 @@ aiofiles==23.2.1
 altair==5.2.0
 annotated-types==0.6.0
 anyio==4.2.0
+apscheduler==3.10.4
 argilla==1.23.0
 attrs==23.2.0
 backoff==2.2.1