add run files

app.py

@@ -16,7 +16,7 @@ AFRIXNLI_TRANSLATE = "afrixnli_translate"
 
 BENCHMARKS = [AFRIMMLU_DIRECT, AFRIMMLU_TRANSLATE, AFRIXNLI_DIRECT, AFRIXNLI_TRANSLATE]
 
-METRICS = ["acc_norm", "acc_norm", "acc_norm"]
+METRICS = ["acc", "acc_stderr", "f1"]
 
 LANGS = ['amh', 'eng', 'ewe', 'fra', 'hau', 'ibo', 'kin', 'lin', 'lug', 'orm', 'sna', 'sot', 'swa', 'twi', 'wol', 'xho', 'yor', 'zul']
 
@@ -66,8 +66,6 @@ def collect_results():
         pretrained_models.add(pretrained)
 
         for lang_task, perfs in results.items():
-            print(lang_task)
-            print(perfs)
             lang_task = lang_task.split('_')
             lang = lang_task[-1]
             task = '_'.join(lang_task[:-1])
@@ -75,7 +73,6 @@ def collect_results():
             assert task in BENCHMARKS
 
             if lang and task:
-                print(BENCHMARKS.index(task))
                 metric = METRICS[BENCHMARKS.index(task)-1]
                 p = round(perfs[metric] * 100, 1)
                 performance_dict[(pretrained, lang)][task] = p

evals/afrimmlu_direct/afrimmlu_direct_amh-Meta-Llama-3-8B-Instruct.json

@@ -1,23 +1,25 @@
 {
-    "results": {
-      "afrimmlu_direct_amh": {
-        "acc": 0.2634730538922156,
-        "acc_stderr": 0.012889646336321774,
-        "acc_norm": 0.31394354148845166,
-        "acc_norm_stderr": 0.013579515768185788
-      }
-    },
-    "versions": {
-      "afrimmlu_direct_amh": 0
-    },
-    "config": {
-      "model": "hf",
-      "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
-      "batch_size": 8,
-      "device": "cuda",
-      "no_cache": false,
-      "limit": null,
-      "bootstrap_iters": 100000,
-      "description_dict": {}
+  "results": {
+    "afrimmlu_direct_amh": {
+      "acc": 0.294,
+      "acc_stderr": 0.02039509548493662,
+      "f1": 0.2746897269174172,
+      "f1_stderr": "N/A"
     }
-  }
+  },
+  "versions": {
+    "afrimmlu_direct_amh": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_amh-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_amh": {
+      "acc": 0.198,
+      "acc_stderr": 0.01783895896384723,
+      "f1": 0.17909485060936567,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_amh": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_eng-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_eng": {
+      "acc": 0.54,
+      "acc_stderr": 0.022311333245289663,
+      "f1": 0.5382451731126852,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_eng": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_eng-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_eng": {
+      "acc": 0.202,
+      "acc_stderr": 0.017973260031288258,
+      "f1": 0.1810217396082914,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_eng": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_ewe-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_ewe": {
+      "acc": 0.256,
+      "acc_stderr": 0.019536923574747615,
+      "f1": 0.25092528422459087,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_ewe": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_ewe-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_ewe": {
+      "acc": 0.216,
+      "acc_stderr": 0.018421909061411935,
+      "f1": 0.2090649866201732,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_ewe": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_fra-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_fra": {
+      "acc": 0.52,
+      "acc_stderr": 0.02236516042423133,
+      "f1": 0.5208080267558528,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_fra": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_fra-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_fra": {
+      "acc": 0.214,
+      "acc_stderr": 0.01835979750238705,
+      "f1": 0.19892885512438008,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_fra": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_hau-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_hau": {
+      "acc": 0.284,
+      "acc_stderr": 0.02018670369357086,
+      "f1": 0.28010976427568457,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_hau": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_hau-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_hau": {
+      "acc": 0.198,
+      "acc_stderr": 0.01783895896384723,
+      "f1": 0.18262451497980564,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_hau": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_ibo-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_ibo": {
+      "acc": 0.33,
+      "acc_stderr": 0.02104961216613481,
+      "f1": 0.3245851895177857,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_ibo": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_ibo-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_ibo": {
+      "acc": 0.216,
+      "acc_stderr": 0.018421909061411935,
+      "f1": 0.19962216343021852,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_ibo": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_kin-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_kin": {
+      "acc": 0.296,
+      "acc_stderr": 0.020435342091896132,
+      "f1": 0.28675065419514817,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_kin": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_kin-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_kin": {
+      "acc": 0.196,
+      "acc_stderr": 0.017770751227744862,
+      "f1": 0.1757764061374706,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_kin": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_lin-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_lin": {
+      "acc": 0.322,
+      "acc_stderr": 0.02091666833001988,
+      "f1": 0.32402662210255406,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_lin": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_lin-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_lin": {
+      "acc": 0.206,
+      "acc_stderr": 0.018104794037333578,
+      "f1": 0.1892508672233748,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_lin": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_lug-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_lug": {
+      "acc": 0.274,
+      "acc_stderr": 0.019966103540279462,
+      "f1": 0.26961388059171515,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_lug": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_lug-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_lug": {
+      "acc": 0.186,
+      "acc_stderr": 0.01741880678058393,
+      "f1": 0.1659062113821138,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_lug": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_orm-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_orm": {
+      "acc": 0.29,
+      "acc_stderr": 0.02031317923174519,
+      "f1": 0.27651312244521153,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_orm": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_orm-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_orm": {
+      "acc": 0.196,
+      "acc_stderr": 0.017770751227744862,
+      "f1": 0.1850540063965828,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_orm": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_sna-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_sna": {
+      "acc": 0.318,
+      "acc_stderr": 0.020847571620814003,
+      "f1": 0.3122451614597504,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_sna": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_sna-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_sna": {
+      "acc": 0.202,
+      "acc_stderr": 0.017973260031288272,
+      "f1": 0.18269228929246092,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_sna": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_sot-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_sot": {
+      "acc": 0.3,
+      "acc_stderr": 0.020514426225628057,
+      "f1": 0.2929658538402751,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_sot": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_sot-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_sot": {
+      "acc": 0.17,
+      "acc_stderr": 0.01681563353139343,
+      "f1": 0.14781723073525851,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_sot": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_swa-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_swa": {
+      "acc": 0.338,
+      "acc_stderr": 0.02117566569520941,
+      "f1": 0.32951382759209963,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_swa": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_swa-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_swa": {
+      "acc": 0.2,
+      "acc_stderr": 0.01790645924143387,
+      "f1": 0.1770651960776954,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_swa": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_twi-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_twi": {
+      "acc": 0.298,
+      "acc_stderr": 0.020475118092988964,
+      "f1": 0.2871954716168561,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_twi": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_twi-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_twi": {
+      "acc": 0.236,
+      "acc_stderr": 0.01900869962208472,
+      "f1": 0.22301842452462128,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_twi": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_wol-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_wol": {
+      "acc": 0.294,
+      "acc_stderr": 0.02039509548493661,
+      "f1": 0.28966157375341367,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_wol": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_wol-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_wol": {
+      "acc": 0.21,
+      "acc_stderr": 0.018233620865305916,
+      "f1": 0.19254414015593024,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_wol": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_xho-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_xho": {
+      "acc": 0.28,
+      "acc_stderr": 0.020099950647503237,
+      "f1": 0.2719845000284934,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_xho": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_xho-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_xho": {
+      "acc": 0.214,
+      "acc_stderr": 0.01835979750238703,
+      "f1": 0.1970288455340002,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_xho": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_yor-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_yor": {
+      "acc": 0.284,
+      "acc_stderr": 0.02018670369357086,
+      "f1": 0.2753758430338612,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_yor": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_yor-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_yor": {
+      "acc": 0.21,
+      "acc_stderr": 0.018233620865305916,
+      "f1": 0.1953916524581152,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_yor": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_zul-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_zul": {
+      "acc": 0.288,
+      "acc_stderr": 0.020271503835075217,
+      "f1": 0.2879464935505291,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_zul": 1.0,
+    "wandb_run_name": "hopeful-vortex-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_direct/afrimmlu_direct_zul-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_direct_zul": {
+      "acc": 0.21,
+      "acc_stderr": 0.018233620865305916,
+      "f1": 0.1859078146090128,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_direct_zul": 1.0,
+    "wandb_run_name": "cool-river-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_amh-Meta-Llama-3-8B-Instruct.json

@@ -1,23 +0,0 @@
-{
-    "results": {
-      "afrimmlu_translate_amh": {
-        "acc": 0.2634730538922156,
-        "acc_stderr": 0.012889646336321774,
-        "acc_norm": 0.31394354148845166,
-        "acc_norm_stderr": 0.013579515768185788
-      }
-    },
-    "versions": {
-      "afrimmlu_translate_amh": 0
-    },
-    "config": {
-      "model": "hf",
-      "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
-      "batch_size": 8,
-      "device": "cuda",
-      "no_cache": false,
-      "limit": null,
-      "bootstrap_iters": 100000,
-      "description_dict": {}
-    }
-  }

evals/afrixnli_direct/afrixnli_direct_amh-Meta-Llama-3-8B-Instruct.json

@@ -1,23 +0,0 @@
-{
-    "results": {
-      "afrixnli_direct_amh": {
-        "acc": 0.2634730538922156,
-        "acc_stderr": 0.012889646336321774,
-        "acc_norm": 0.31394354148845166,
-        "acc_norm_stderr": 0.013579515768185788
-      }
-    },
-    "versions": {
-      "afrixnli_direct_amh": 0
-    },
-    "config": {
-      "model": "hf",
-      "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
-      "batch_size": 8,
-      "device": "cuda",
-      "no_cache": false,
-      "limit": null,
-      "bootstrap_iters": 100000,
-      "description_dict": {}
-    }
-  }

evals/afrixnli_translate/afrixnli_translate_amh-Meta-Llama-3-8B-Instruct.json

@@ -1,23 +0,0 @@
-{
-    "results": {
-      "afrixnli_translate_amh": {
-        "acc": 0.2634730538922156,
-        "acc_stderr": 0.012889646336321774,
-        "acc_norm": 0.31394354148845166,
-        "acc_norm_stderr": 0.013579515768185788
-      }
-    },
-    "versions": {
-      "afrixnli_translate_amh": 0
-    },
-    "config": {
-      "model": "hf",
-      "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
-      "batch_size": 8,
-      "device": "cuda",
-      "no_cache": false,
-      "limit": null,
-      "bootstrap_iters": 100000,
-      "description_dict": {}
-    }
-  }

pull_benchmark_data.py

@@ -0,0 +1,69 @@
+import os
+import json
+import wandb
+import argparse
+
+curr_dir = os.path.dirname(os.path.realpath(__file__))
+
+LANGUAGES = ['amh', 'eng', 'ewe', 'fra', 'hau', 'ibo', 'kin', 'lin', 'lug', 'orm', 'sna', 'sot', 'swa', 'twi', 'wol', 'xho', 'yor', 'zul']
+BENCHMARCK2PROJECT = {
+    "afrimmlu_direct" : "african-research-collective/llm-evaluation-afrimmlu-direct"
+}
+BENCHMARK_VERSION = 1.0
+
+
+
+def main(args):
+    api = wandb.Api()
+
+    runs = api.runs(BENCHMARCK2PROJECT[args.benchmark])
+
+    print(runs)
+
+    for run in runs: 
+        # .summary contains the output keys/values for metrics like accuracy.
+        #  We call ._json_dict to omit large files
+
+        for lang in LANGUAGES:
+            lang_result_key = f'{args.benchmark}_{lang}'
+
+            results = {lang_result_key: {}}
+            config = {}
+            versions = {}
+
+            
+            results[lang_result_key]['acc'] = run.summary._json_dict[f'{lang_result_key}/acc']
+            results[lang_result_key]['acc_stderr'] = run.summary._json_dict[f'{lang_result_key}/acc_stderr']
+            results[lang_result_key]['f1'] = run.summary._json_dict[f'{lang_result_key}/f1']
+            results[lang_result_key]['f1_stderr'] = run.summary._json_dict[f'{lang_result_key}/f1_stderr']
+
+            versions[lang_result_key] = BENCHMARK_VERSION
+            versions['wandb_run_name'] = run.name
+
+            config['model'] = run.config['cli_configs']['model']
+            config['model_args'] = run.config['cli_configs']['model_args']
+            config['batch_size'] = run.config['cli_configs']['batch_size']
+            config['device'] = run.config['cli_configs']['device']
+            config['model_dtype'] = run.config['cli_configs']['model_dtype']
+            config['numpy_seed'] = run.config['cli_configs']['numpy_seed']
+            config['torch_seed'] = run.config['cli_configs']['torch_seed']
+            config['random_seed'] = run.config['cli_configs']['random_seed']
+            config['fewshot_seed'] = run.config['cli_configs']['fewshot_seed']
+
+            final_json_object = {
+                'results': results,
+                'versions': versions,
+                'config': config
+            }
+
+            pretrained_model = config['model_args'].split(',')[0].split('=')[1].split('/')[-1]
+
+            with open(os.path.join(curr_dir, f"evals/{args.benchmark}/{args.benchmark}_{lang}-{pretrained_model}.json"), 'w') as f:
+                json.dump(final_json_object, f, indent=2)
+            
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--benchmark', type=str, required=True)
+    args = parser.parse_args()
+    main(args)