Update
Browse files- update_scheduler.py +5 -4
update_scheduler.py
CHANGED
@@ -45,10 +45,11 @@ class RepoUpdater:
|
|
45 |
api = HfApi()
|
46 |
name = api.whoami()['name']
|
47 |
|
48 |
-
|
49 |
tempfile.tempdir) / repo_id.split('/')[-1] # type: ignore
|
|
|
50 |
self.repo = Repository(
|
51 |
-
local_dir=
|
52 |
clone_from=repo_id,
|
53 |
repo_type=repo_type,
|
54 |
git_user=name,
|
@@ -66,7 +67,7 @@ class RepoUpdater:
|
|
66 |
f'https://huggingface.co/api/daily_papers?date={today}').json()
|
67 |
|
68 |
self.repo.git_pull()
|
69 |
-
df = pd.read_csv(self.
|
70 |
rows = [row for _, row in df.iterrows()]
|
71 |
arxiv_ids = {row.arxiv_id for row in rows}
|
72 |
|
@@ -84,7 +85,7 @@ class RepoUpdater:
|
|
84 |
'github': github,
|
85 |
}))
|
86 |
df = pd.DataFrame(rows).reset_index(drop=True)
|
87 |
-
df.to_csv(self.
|
88 |
|
89 |
def push(self) -> None:
|
90 |
self.repo.push_to_hub()
|
|
|
45 |
api = HfApi()
|
46 |
name = api.whoami()['name']
|
47 |
|
48 |
+
repo_dir = pathlib.Path(
|
49 |
tempfile.tempdir) / repo_id.split('/')[-1] # type: ignore
|
50 |
+
self.csv_path = repo_dir / 'papers.csv'
|
51 |
self.repo = Repository(
|
52 |
+
local_dir=repo_dir,
|
53 |
clone_from=repo_id,
|
54 |
repo_type=repo_type,
|
55 |
git_user=name,
|
|
|
67 |
f'https://huggingface.co/api/daily_papers?date={today}').json()
|
68 |
|
69 |
self.repo.git_pull()
|
70 |
+
df = pd.read_csv(self.csv_path, dtype=str).fillna('')
|
71 |
rows = [row for _, row in df.iterrows()]
|
72 |
arxiv_ids = {row.arxiv_id for row in rows}
|
73 |
|
|
|
85 |
'github': github,
|
86 |
}))
|
87 |
df = pd.DataFrame(rows).reset_index(drop=True)
|
88 |
+
df.to_csv(self.csv_path, index=False)
|
89 |
|
90 |
def push(self) -> None:
|
91 |
self.repo.push_to_hub()
|