lewtun HF staff commited on
Commit
a0e78e7
β€’
1 Parent(s): 580b4e4

Add cron job

Browse files
.github/workflows/run_evaluation_jobs.yml ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: Start evaluation jobs
2
+
3
+ on:
4
+ schedule:
5
+ - cron: '*/15 * * * *' # Start evaluations every 15th minute
6
+
7
+ jobs:
8
+
9
+ build:
10
+ runs-on: ubuntu-latest
11
+
12
+ steps:
13
+ - name: Checkout code
14
+ uses: actions/checkout@v2
15
+
16
+ - name: Setup Python Environment
17
+ uses: actions/setup-python@v2
18
+ with:
19
+ python-version: 3.8
20
+
21
+ - name: Install requirements
22
+ run: pip install -r requirements.txt
23
+
24
+ - name: Execute scoring script
25
+ env:
26
+ HF_TOKEN: ${{ secrets.HF_GEM_TOKEN }}
27
+ run: |
28
+ HF_TOKEN=$HF_TOKEN AUTOTRAIN_USERNAME=$AUTOTRAIN_USERNAME AUTOTRAIN_BACKEND_API=$AUTOTRAIN_BACKEND_API python run_evaluation_jobs.py
run_evaluation_jobs.py CHANGED
@@ -12,7 +12,6 @@ if Path(".env").is_file():
12
  load_dotenv(".env")
13
 
14
  HF_TOKEN = os.getenv("HF_TOKEN")
15
- AUTOTRAIN_TOKEN = os.getenv("AUTOTRAIN_TOKEN")
16
  AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME")
17
  AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API")
18
 
@@ -24,15 +23,15 @@ else:
24
 
25
  def main():
26
  logs_df = load_dataset("autoevaluate/evaluation-job-logs", use_auth_token=True, split="train").to_pandas()
27
- # Filter out
28
- # - legacy AutoTrain submissions prior to project approvals was implemented
29
- # - submissions for appropriate AutoTrain environment (staging vs prod)
30
- projects_df = logs_df.copy()[
31
- (~logs_df["project_id"].isnull()) & (logs_df.query(f"autotrain_env == '{AUTOTRAIN_ENV}'"))
32
- ]
33
  projects_to_approve = projects_df["project_id"].astype(int).tolist()
 
34
 
35
  for project_id in projects_to_approve:
 
36
  try:
37
  project_info = http_get(
38
  path=f"/projects/{project_id}",
@@ -46,8 +45,9 @@ def main():
46
  domain=AUTOTRAIN_BACKEND_API,
47
  ).json()
48
  print(f"πŸƒβ€β™‚οΈ Project {project_id} approval response: {train_job_resp}")
49
- except:
50
  print(f"There was a problem obtaining the project info for project ID {project_id}")
 
51
  pass
52
 
53
 
 
12
  load_dotenv(".env")
13
 
14
  HF_TOKEN = os.getenv("HF_TOKEN")
 
15
  AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME")
16
  AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API")
17
 
 
23
 
24
  def main():
25
  logs_df = load_dataset("autoevaluate/evaluation-job-logs", use_auth_token=True, split="train").to_pandas()
26
+ # Filter out legacy AutoTrain submissions prior to project approvals requirement
27
+ projects_df = logs_df.copy()[(~logs_df["project_id"].isnull())]
28
+ # Filter IDs for appropriate AutoTrain env (staging vs prod)
29
+ projects_df = projects_df.copy().query(f"autotrain_env == '{AUTOTRAIN_ENV}'")
 
 
30
  projects_to_approve = projects_df["project_id"].astype(int).tolist()
31
+ print(f"πŸš€ Found {len(projects_to_approve)} evaluation projects to approve!")
32
 
33
  for project_id in projects_to_approve:
34
+ print(f"Attempting to evaluate project ID {project_id} ...")
35
  try:
36
  project_info = http_get(
37
  path=f"/projects/{project_id}",
 
45
  domain=AUTOTRAIN_BACKEND_API,
46
  ).json()
47
  print(f"πŸƒβ€β™‚οΈ Project {project_id} approval response: {train_job_resp}")
48
+ except Exception as e:
49
  print(f"There was a problem obtaining the project info for project ID {project_id}")
50
+ print(f"Error message: {e}")
51
  pass
52
 
53