Use apscheduler to update the index file

papers.py

@@ -4,20 +4,39 @@ import operator
 import datasets
 import pandas as pd
 import tqdm.auto
+from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import HfApi
 from ragatouille import RAGPretrainedModel
 
 api = HfApi()
 
+INDEX_REPO_ID = "hysts-bot-data/daily-papers-abstract-index"
 INDEX_DIR_PATH = ".ragatouille/colbert/indexes/daily-papers-abstract-index/"
 api.snapshot_download(
-    repo_id="hysts-bot-data/daily-papers-abstract-index",
+    repo_id=INDEX_REPO_ID,
     repo_type="dataset",
     local_dir=INDEX_DIR_PATH,
 )
-ABSTRACT_RETRIEVER = RAGPretrainedModel.from_index(INDEX_DIR_PATH)
+abstract_retriever = RAGPretrainedModel.from_index(INDEX_DIR_PATH)
 # Run once to initialize the retriever
-ABSTRACT_RETRIEVER.search("LLM")
+abstract_retriever.search("LLM")
+
+
+def update_abstract_index() -> None:
+    global abstract_retriever
+
+    api.snapshot_download(
+        repo_id=INDEX_REPO_ID,
+        repo_type="dataset",
+        local_dir=INDEX_DIR_PATH,
+    )
+    abstract_retriever = RAGPretrainedModel.from_index(INDEX_DIR_PATH)
+    abstract_retriever.search("LLM")
+
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(func=update_abstract_index, trigger="cron", hour="*", timezone="UTC", misfire_grace_time=3 * 60)
+scheduler.start()
 
 
 def get_df() -> pd.DataFrame:
@@ -114,7 +133,7 @@ class PaperList:
 
         # Filter by abstract
         if abstract_search_query:
-            results = ABSTRACT_RETRIEVER.search(abstract_search_query, k=max_num_to_retrieve)
+            results = abstract_retriever.search(abstract_search_query, k=max_num_to_retrieve)
             remaining_ids = set(df["arxiv_id"])
             found_id_set = set()
             found_ids = []

pyproject.toml

@@ -5,6 +5,7 @@ description = ""
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
+    "apscheduler>=3.10.4",
     "datasets>=2.21.0",
     "gradio-calendar>=0.0.4",
     "gradio>=4.44.0",

requirements.txt

@@ -21,6 +21,8 @@ anyio==4.4.0
     #   httpx
     #   openai
     #   starlette
+apscheduler==3.10.4
+    # via daily-papers (pyproject.toml)
 async-timeout==4.0.3
     # via
     #   aiohttp
@@ -406,7 +408,9 @@ python-dotenv==1.0.1
 python-multipart==0.0.9
     # via gradio
 pytz==2024.2
-    # via pandas
+    # via
+    #   apscheduler
+    #   pandas
 pyyaml==6.0.2
     # via
     #   datasets
@@ -455,7 +459,9 @@ setuptools==74.1.2
 shellingham==1.5.4
     # via typer
 six==1.16.0
-    # via python-dateutil
+    # via
+    #   apscheduler
+    #   python-dateutil
 smmap==5.0.1
     # via gitdb
 sniffio==1.3.1
@@ -546,6 +552,8 @@ typing-inspect==0.9.0
     #   llama-index-legacy
 tzdata==2024.1
     # via pandas
+tzlocal==5.2
+    # via apscheduler
 ujson==5.10.0
     # via colbert-ai
 urllib3==2.2.2

uv.lock

@@ -139,6 +139,20 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7b/a2/10639a79341f6c019dedc95bd48a4928eed9f1d1197f4c04f546fc7ae0ff/anyio-4.4.0-py3-none-any.whl", hash = "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7", size = 86780 },
 ]
 
+[[package]]
+name = "apscheduler"
+version = "3.10.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pytz" },
+    { name = "six" },
+    { name = "tzlocal" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/5e/34/5dcb368cf89f93132d9a31bd3747962a9dc874480e54333b0c09fa7d56ac/APScheduler-3.10.4.tar.gz", hash = "sha256:e6df071b27d9be898e486bc7940a7be50b4af2e9da7c08f0744a96d4bd4cef4a", size = 100832 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/13/b5/7af0cb920a476dccd612fbc9a21a3745fb29b1fcd74636078db8f7ba294c/APScheduler-3.10.4-py3-none-any.whl", hash = "sha256:fb91e8a768632a4756a585f79ec834e0e27aad5860bac7eaa523d9ccefd87661", size = 59303 },
+]
+
 [[package]]
 name = "async-timeout"
 version = "4.0.3"
@@ -423,6 +437,7 @@ name = "daily-papers"
 version = "0.1.0"
 source = { virtual = "." }
 dependencies = [
+    { name = "apscheduler" },
     { name = "datasets" },
     { name = "gradio" },
     { name = "gradio-calendar" },
@@ -436,6 +451,7 @@ dependencies = [
 
 [package.metadata]
 requires-dist = [
+    { name = "apscheduler", specifier = ">=3.10.4" },
     { name = "datasets", specifier = ">=2.21.0" },
     { name = "gradio", specifier = ">=4.44.0" },
     { name = "gradio-calendar", specifier = ">=0.0.4" },
@@ -3032,6 +3048,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/65/58/f9c9e6be752e9fcb8b6a0ee9fb87e6e7a1f6bcab2cdc73f02bb7ba91ada0/tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252", size = 345370 },
 ]
 
+[[package]]
+name = "tzlocal"
+version = "5.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "tzdata", marker = "platform_system == 'Windows'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/04/d3/c19d65ae67636fe63953b20c2e4a8ced4497ea232c43ff8d01db16de8dc0/tzlocal-5.2.tar.gz", hash = "sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e", size = 30201 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/97/3f/c4c51c55ff8487f2e6d0e618dba917e3c3ee2caae6cf0fbb59c9b1876f2e/tzlocal-5.2-py3-none-any.whl", hash = "sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8", size = 17859 },
+]
+
 [[package]]
 name = "ujson"
 version = "5.10.0"