Spaces:
Runtime error
Runtime error
Add cron job
Browse files
.github/workflows/run_evaluation_jobs.yml
ADDED
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
name: Start evaluation jobs
|
2 |
+
|
3 |
+
on:
|
4 |
+
schedule:
|
5 |
+
- cron: '*/15 * * * *' # Start evaluations every 15th minute
|
6 |
+
|
7 |
+
jobs:
|
8 |
+
|
9 |
+
build:
|
10 |
+
runs-on: ubuntu-latest
|
11 |
+
|
12 |
+
steps:
|
13 |
+
- name: Checkout code
|
14 |
+
uses: actions/checkout@v2
|
15 |
+
|
16 |
+
- name: Setup Python Environment
|
17 |
+
uses: actions/setup-python@v2
|
18 |
+
with:
|
19 |
+
python-version: 3.8
|
20 |
+
|
21 |
+
- name: Install requirements
|
22 |
+
run: pip install -r requirements.txt
|
23 |
+
|
24 |
+
- name: Execute scoring script
|
25 |
+
env:
|
26 |
+
HF_TOKEN: ${{ secrets.HF_GEM_TOKEN }}
|
27 |
+
run: |
|
28 |
+
HF_TOKEN=$HF_TOKEN AUTOTRAIN_USERNAME=$AUTOTRAIN_USERNAME AUTOTRAIN_BACKEND_API=$AUTOTRAIN_BACKEND_API python run_evaluation_jobs.py
|
run_evaluation_jobs.py
CHANGED
@@ -12,7 +12,6 @@ if Path(".env").is_file():
|
|
12 |
load_dotenv(".env")
|
13 |
|
14 |
HF_TOKEN = os.getenv("HF_TOKEN")
|
15 |
-
AUTOTRAIN_TOKEN = os.getenv("AUTOTRAIN_TOKEN")
|
16 |
AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME")
|
17 |
AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API")
|
18 |
|
@@ -24,15 +23,15 @@ else:
|
|
24 |
|
25 |
def main():
|
26 |
logs_df = load_dataset("autoevaluate/evaluation-job-logs", use_auth_token=True, split="train").to_pandas()
|
27 |
-
# Filter out
|
28 |
-
|
29 |
-
#
|
30 |
-
projects_df =
|
31 |
-
(~logs_df["project_id"].isnull()) & (logs_df.query(f"autotrain_env == '{AUTOTRAIN_ENV}'"))
|
32 |
-
]
|
33 |
projects_to_approve = projects_df["project_id"].astype(int).tolist()
|
|
|
34 |
|
35 |
for project_id in projects_to_approve:
|
|
|
36 |
try:
|
37 |
project_info = http_get(
|
38 |
path=f"/projects/{project_id}",
|
@@ -46,8 +45,9 @@ def main():
|
|
46 |
domain=AUTOTRAIN_BACKEND_API,
|
47 |
).json()
|
48 |
print(f"πββοΈ Project {project_id} approval response: {train_job_resp}")
|
49 |
-
except:
|
50 |
print(f"There was a problem obtaining the project info for project ID {project_id}")
|
|
|
51 |
pass
|
52 |
|
53 |
|
|
|
12 |
load_dotenv(".env")
|
13 |
|
14 |
HF_TOKEN = os.getenv("HF_TOKEN")
|
|
|
15 |
AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME")
|
16 |
AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API")
|
17 |
|
|
|
23 |
|
24 |
def main():
|
25 |
logs_df = load_dataset("autoevaluate/evaluation-job-logs", use_auth_token=True, split="train").to_pandas()
|
26 |
+
# Filter out legacy AutoTrain submissions prior to project approvals requirement
|
27 |
+
projects_df = logs_df.copy()[(~logs_df["project_id"].isnull())]
|
28 |
+
# Filter IDs for appropriate AutoTrain env (staging vs prod)
|
29 |
+
projects_df = projects_df.copy().query(f"autotrain_env == '{AUTOTRAIN_ENV}'")
|
|
|
|
|
30 |
projects_to_approve = projects_df["project_id"].astype(int).tolist()
|
31 |
+
print(f"π Found {len(projects_to_approve)} evaluation projects to approve!")
|
32 |
|
33 |
for project_id in projects_to_approve:
|
34 |
+
print(f"Attempting to evaluate project ID {project_id} ...")
|
35 |
try:
|
36 |
project_info = http_get(
|
37 |
path=f"/projects/{project_id}",
|
|
|
45 |
domain=AUTOTRAIN_BACKEND_API,
|
46 |
).json()
|
47 |
print(f"πββοΈ Project {project_id} approval response: {train_job_resp}")
|
48 |
+
except Exception as e:
|
49 |
print(f"There was a problem obtaining the project info for project ID {project_id}")
|
50 |
+
print(f"Error message: {e}")
|
51 |
pass
|
52 |
|
53 |
|