add mmlu translate

content.py

@@ -3,6 +3,7 @@ TITLE = '<h1 align="center" id="space-title">African Languages LLM Eval Leaderbo
 INTRO_TEXT = f"""
 ## About
 This leaderboard tracks progress and ranks performance of large language models (LLMs) on African languages.
+This project uses the [lm-evaluation-harness by EleutherAI](https://github.com/EleutherAI/lm-evaluation-harness) for evaluation, focusing on African language tasks. Some of the tasks contained in this leaderboard have already been built into the harness (e.g. IrokoBench Tasks & Belebele).
 
 We currently evaluate models over the following benchmarks:
 

evals/afrimmlu_translate/afrimmlu_translate_amh-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_amh": {
+      "acc": 0.394,
+      "acc_stderr": 0.021874299301689253,
+      "f1": 0.3911411480086645,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_amh": 1.0,
+    "wandb_run_name": "happy-fog-3"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_amh-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_amh": {
+      "acc": 0.212,
+      "acc_stderr": 0.01829703700401389,
+      "f1": 0.193400184674494,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_amh": 1.0,
+    "wandb_run_name": "cool-moon-2"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_amh-aya-101.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_amh": {
+      "acc": 0.32,
+      "acc_stderr": 0.020882340488761805,
+      "f1": 0.321055726365249,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_amh": 1.0,
+    "wandb_run_name": "major-rain-4"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=CohereForAI/aya-101,parallelize=True",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_amh-bloomz-1b7.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_amh": {
+      "acc": 0.232,
+      "acc_stderr": 0.018896193591952055,
+      "f1": 0.2303475443171238,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_amh": 1.0,
+    "wandb_run_name": "amber-sponge-5"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/bloomz-1b7",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_amh-bloomz-3b.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_amh": {
+      "acc": 0.252,
+      "acc_stderr": 0.019435727282249543,
+      "f1": 0.2511004641341941,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_amh": 1.0,
+    "wandb_run_name": "atomic-lake-6"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/bloomz-3b",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_amh-bloomz-7b1.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_amh": {
+      "acc": 0.264,
+      "acc_stderr": 0.019732885585922084,
+      "f1": 0.2611884211157269,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_amh": 1.0,
+    "wandb_run_name": "silver-moon-7"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/bloomz-7b1",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_amh-flan-t5-base.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_amh": {
+      "acc": 0.23,
+      "acc_stderr": 0.01883905039112313,
+      "f1": 0.22635667655752564,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_amh": 1.0,
+    "wandb_run_name": "serene-frost-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=google/flan-t5-base",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_amh-flan-t5-large.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_amh": {
+      "acc": 0.306,
+      "acc_stderr": 0.020629569998345417,
+      "f1": 0.30646879281309314,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_amh": 1.0,
+    "wandb_run_name": "whole-serenity-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=google/flan-t5-large",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_amh-flan-t5-small.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_amh": {
+      "acc": 0.26,
+      "acc_stderr": 0.019635965529725512,
+      "f1": 0.25428716687048475,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_amh": 1.0,
+    "wandb_run_name": "fresh-moon-14"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=google/flan-t5-small",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_amh-mt0-base.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_amh": {
+      "acc": 0.242,
+      "acc_stderr": 0.019173085678337178,
+      "f1": 0.23588653079268743,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_amh": 1.0,
+    "wandb_run_name": "icy-star-8"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/mt0-base",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_amh-mt0-large.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_amh": {
+      "acc": 0.252,
+      "acc_stderr": 0.01943572728224953,
+      "f1": 0.2492758416335272,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_amh": 1.0,
+    "wandb_run_name": "light-cherry-9"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/mt0-large",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_amh-mt0-small.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_amh": {
+      "acc": 0.228,
+      "acc_stderr": 0.018781306529363197,
+      "f1": 0.21952920076616783,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_amh": 1.0,
+    "wandb_run_name": "brisk-sun-10"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/mt0-small",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_amh-mt0-xl.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_amh": {
+      "acc": 0.318,
+      "acc_stderr": 0.020847571620814017,
+      "f1": 0.3106228371529179,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_amh": 1.0,
+    "wandb_run_name": "soft-dew-11"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/mt0-xl",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ewe-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ewe": {
+      "acc": 0.338,
+      "acc_stderr": 0.021175665695209407,
+      "f1": 0.33931031230441916,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ewe": 1.0,
+    "wandb_run_name": "happy-fog-3"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ewe-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ewe": {
+      "acc": 0.222,
+      "acc_stderr": 0.01860441475825008,
+      "f1": 0.21571520023861815,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ewe": 1.0,
+    "wandb_run_name": "cool-moon-2"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ewe-aya-101.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ewe": {
+      "acc": 0.304,
+      "acc_stderr": 0.02059164957122493,
+      "f1": 0.2984881378963239,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ewe": 1.0,
+    "wandb_run_name": "major-rain-4"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=CohereForAI/aya-101,parallelize=True",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ewe-bloomz-1b7.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ewe": {
+      "acc": 0.244,
+      "acc_stderr": 0.0192267348936146,
+      "f1": 0.24428173105444703,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ewe": 1.0,
+    "wandb_run_name": "amber-sponge-5"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/bloomz-1b7",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ewe-bloomz-3b.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ewe": {
+      "acc": 0.246,
+      "acc_stderr": 0.019279819056352548,
+      "f1": 0.2468667245003703,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ewe": 1.0,
+    "wandb_run_name": "atomic-lake-6"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/bloomz-3b",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ewe-bloomz-7b1.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ewe": {
+      "acc": 0.272,
+      "acc_stderr": 0.01992048320956609,
+      "f1": 0.27134200495691096,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ewe": 1.0,
+    "wandb_run_name": "silver-moon-7"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/bloomz-7b1",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ewe-flan-t5-base.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ewe": {
+      "acc": 0.274,
+      "acc_stderr": 0.01996610354027947,
+      "f1": 0.2733405472168771,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ewe": 1.0,
+    "wandb_run_name": "serene-frost-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=google/flan-t5-base",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ewe-flan-t5-large.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ewe": {
+      "acc": 0.256,
+      "acc_stderr": 0.019536923574747612,
+      "f1": 0.25476798405989093,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ewe": 1.0,
+    "wandb_run_name": "whole-serenity-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=google/flan-t5-large",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ewe-flan-t5-small.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ewe": {
+      "acc": 0.26,
+      "acc_stderr": 0.019635965529725512,
+      "f1": 0.24971217821262703,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ewe": 1.0,
+    "wandb_run_name": "fresh-moon-14"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=google/flan-t5-small",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ewe-mt0-base.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ewe": {
+      "acc": 0.226,
+      "acc_stderr": 0.018722956449139933,
+      "f1": 0.21750280479645925,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ewe": 1.0,
+    "wandb_run_name": "icy-star-8"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/mt0-base",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ewe-mt0-large.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ewe": {
+      "acc": 0.238,
+      "acc_stderr": 0.01906407295819844,
+      "f1": 0.23136212205719056,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ewe": 1.0,
+    "wandb_run_name": "light-cherry-9"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/mt0-large",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ewe-mt0-small.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ewe": {
+      "acc": 0.248,
+      "acc_stderr": 0.019332342821239107,
+      "f1": 0.23917170024216244,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ewe": 1.0,
+    "wandb_run_name": "brisk-sun-10"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/mt0-small",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ewe-mt0-xl.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ewe": {
+      "acc": 0.29,
+      "acc_stderr": 0.02031317923174519,
+      "f1": 0.28795219737987676,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ewe": 1.0,
+    "wandb_run_name": "soft-dew-11"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/mt0-xl",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_hau-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_hau": {
+      "acc": 0.36,
+      "acc_stderr": 0.02148775108972053,
+      "f1": 0.3604494892242261,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_hau": 1.0,
+    "wandb_run_name": "happy-fog-3"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_hau-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_hau": {
+      "acc": 0.24,
+      "acc_stderr": 0.01911886665375974,
+      "f1": 0.2223787592766505,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_hau": 1.0,
+    "wandb_run_name": "cool-moon-2"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_hau-aya-101.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_hau": {
+      "acc": 0.308,
+      "acc_stderr": 0.0206670329874661,
+      "f1": 0.3039058851447556,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_hau": 1.0,
+    "wandb_run_name": "major-rain-4"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=CohereForAI/aya-101,parallelize=True",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_hau-bloomz-1b7.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_hau": {
+      "acc": 0.274,
+      "acc_stderr": 0.019966103540279445,
+      "f1": 0.2724977541568708,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_hau": 1.0,
+    "wandb_run_name": "amber-sponge-5"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/bloomz-1b7",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_hau-bloomz-3b.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_hau": {
+      "acc": 0.304,
+      "acc_stderr": 0.020591649571224932,
+      "f1": 0.3034697818886015,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_hau": 1.0,
+    "wandb_run_name": "atomic-lake-6"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/bloomz-3b",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_hau-bloomz-7b1.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_hau": {
+      "acc": 0.32,
+      "acc_stderr": 0.020882340488761805,
+      "f1": 0.3200184974536263,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_hau": 1.0,
+    "wandb_run_name": "silver-moon-7"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/bloomz-7b1",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_hau-flan-t5-base.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_hau": {
+      "acc": 0.248,
+      "acc_stderr": 0.019332342821239103,
+      "f1": 0.24470006662225183,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_hau": 1.0,
+    "wandb_run_name": "serene-frost-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=google/flan-t5-base",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_hau-flan-t5-large.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_hau": {
+      "acc": 0.29,
+      "acc_stderr": 0.020313179231745207,
+      "f1": 0.2906470359611149,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_hau": 1.0,
+    "wandb_run_name": "whole-serenity-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=google/flan-t5-large",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_hau-flan-t5-small.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_hau": {
+      "acc": 0.278,
+      "acc_stderr": 0.020055833888070897,
+      "f1": 0.27432208144412573,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_hau": 1.0,
+    "wandb_run_name": "fresh-moon-14"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=google/flan-t5-small",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_hau-mt0-base.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_hau": {
+      "acc": 0.224,
+      "acc_stderr": 0.01866399446471079,
+      "f1": 0.21520681120974008,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_hau": 1.0,
+    "wandb_run_name": "icy-star-8"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/mt0-base",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_hau-mt0-large.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_hau": {
+      "acc": 0.244,
+      "acc_stderr": 0.01922673489361459,
+      "f1": 0.2370237396867334,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_hau": 1.0,
+    "wandb_run_name": "light-cherry-9"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/mt0-large",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_hau-mt0-small.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_hau": {
+      "acc": 0.22,
+      "acc_stderr": 0.01854421137582033,
+      "f1": 0.2165969551393627,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_hau": 1.0,
+    "wandb_run_name": "brisk-sun-10"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/mt0-small",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_hau-mt0-xl.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_hau": {
+      "acc": 0.252,
+      "acc_stderr": 0.01943572728224952,
+      "f1": 0.24627743028468996,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_hau": 1.0,
+    "wandb_run_name": "soft-dew-11"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/mt0-xl",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ibo-Meta-Llama-3-8B-Instruct.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ibo": {
+      "acc": 0.364,
+      "acc_stderr": 0.02153917063731769,
+      "f1": 0.3620861488890409,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ibo": 1.0,
+    "wandb_run_name": "happy-fog-3"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Meta-Llama-3-8B-Instruct",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.bfloat16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ibo-afriteva_v2_large_ayaft.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ibo": {
+      "acc": 0.214,
+      "acc_stderr": 0.018359797502387025,
+      "f1": 0.19723354975358284,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ibo": 1.0,
+    "wandb_run_name": "cool-moon-2"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=taresco/afriteva_v2_large_ayaft",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ibo-aya-101.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ibo": {
+      "acc": 0.316,
+      "acc_stderr": 0.020812359515855857,
+      "f1": 0.31581378648091524,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ibo": 1.0,
+    "wandb_run_name": "major-rain-4"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=CohereForAI/aya-101,parallelize=True",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ibo-bloomz-1b7.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ibo": {
+      "acc": 0.238,
+      "acc_stderr": 0.019064072958198452,
+      "f1": 0.24129061923077083,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ibo": 1.0,
+    "wandb_run_name": "amber-sponge-5"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/bloomz-1b7",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ibo-bloomz-3b.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ibo": {
+      "acc": 0.252,
+      "acc_stderr": 0.01943572728224953,
+      "f1": 0.2521675138674737,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ibo": 1.0,
+    "wandb_run_name": "atomic-lake-6"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/bloomz-3b",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ibo-bloomz-7b1.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ibo": {
+      "acc": 0.252,
+      "acc_stderr": 0.01943572728224953,
+      "f1": 0.2527994546842154,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ibo": 1.0,
+    "wandb_run_name": "silver-moon-7"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/bloomz-7b1",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float16",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ibo-flan-t5-base.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ibo": {
+      "acc": 0.264,
+      "acc_stderr": 0.01973288558592211,
+      "f1": 0.26186909736627434,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ibo": 1.0,
+    "wandb_run_name": "serene-frost-12"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=google/flan-t5-base",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ibo-flan-t5-large.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ibo": {
+      "acc": 0.294,
+      "acc_stderr": 0.020395095484936617,
+      "f1": 0.2930675969903471,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ibo": 1.0,
+    "wandb_run_name": "whole-serenity-13"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=google/flan-t5-large",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ibo-flan-t5-small.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ibo": {
+      "acc": 0.262,
+      "acc_stderr": 0.019684688820194703,
+      "f1": 0.2533535628301984,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ibo": 1.0,
+    "wandb_run_name": "fresh-moon-14"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=google/flan-t5-small",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}

evals/afrimmlu_translate/afrimmlu_translate_ibo-mt0-base.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "afrimmlu_translate_ibo": {
+      "acc": 0.25,
+      "acc_stderr": 0.019384310743640384,
+      "f1": 0.24592729926179152,
+      "f1_stderr": "N/A"
+    }
+  },
+  "versions": {
+    "afrimmlu_translate_ibo": 1.0,
+    "wandb_run_name": "icy-star-8"
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=bigscience/mt0-base",
+    "batch_size": 8,
+    "device": null,
+    "model_dtype": "torch.float32",
+    "numpy_seed": 42,
+    "torch_seed": 42,
+    "random_seed": 42,
+    "fewshot_seed": 42
+  }
+}