|
--- |
|
license: apache-2.0 |
|
library_name: transformers |
|
tags: |
|
- merge |
|
pipeline_tag: text-generation |
|
model-index: |
|
- name: TheTop-5x7B-Instruct-P-v0.1 |
|
results: |
|
- task: |
|
type: text-generation |
|
name: Text Generation |
|
dataset: |
|
name: AI2 Reasoning Challenge (25-Shot) |
|
type: ai2_arc |
|
config: ARC-Challenge |
|
split: test |
|
args: |
|
num_few_shot: 25 |
|
metrics: |
|
- type: acc_norm |
|
value: 38.57 |
|
name: normalized accuracy |
|
source: |
|
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=MaziyarPanahi/TheTop-5x7B-Instruct-P-v0.1 |
|
name: Open LLM Leaderboard |
|
- task: |
|
type: text-generation |
|
name: Text Generation |
|
dataset: |
|
name: HellaSwag (10-Shot) |
|
type: hellaswag |
|
split: validation |
|
args: |
|
num_few_shot: 10 |
|
metrics: |
|
- type: acc_norm |
|
value: 51.54 |
|
name: normalized accuracy |
|
source: |
|
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=MaziyarPanahi/TheTop-5x7B-Instruct-P-v0.1 |
|
name: Open LLM Leaderboard |
|
- task: |
|
type: text-generation |
|
name: Text Generation |
|
dataset: |
|
name: MMLU (5-Shot) |
|
type: cais/mmlu |
|
config: all |
|
split: test |
|
args: |
|
num_few_shot: 5 |
|
metrics: |
|
- type: acc |
|
value: 63.36 |
|
name: accuracy |
|
source: |
|
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=MaziyarPanahi/TheTop-5x7B-Instruct-P-v0.1 |
|
name: Open LLM Leaderboard |
|
- task: |
|
type: text-generation |
|
name: Text Generation |
|
dataset: |
|
name: TruthfulQA (0-shot) |
|
type: truthful_qa |
|
config: multiple_choice |
|
split: validation |
|
args: |
|
num_few_shot: 0 |
|
metrics: |
|
- type: mc2 |
|
value: 50.07 |
|
source: |
|
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=MaziyarPanahi/TheTop-5x7B-Instruct-P-v0.1 |
|
name: Open LLM Leaderboard |
|
- task: |
|
type: text-generation |
|
name: Text Generation |
|
dataset: |
|
name: Winogrande (5-shot) |
|
type: winogrande |
|
config: winogrande_xl |
|
split: validation |
|
args: |
|
num_few_shot: 5 |
|
metrics: |
|
- type: acc |
|
value: 72.61 |
|
name: accuracy |
|
source: |
|
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=MaziyarPanahi/TheTop-5x7B-Instruct-P-v0.1 |
|
name: Open LLM Leaderboard |
|
- task: |
|
type: text-generation |
|
name: Text Generation |
|
dataset: |
|
name: GSM8k (5-shot) |
|
type: gsm8k |
|
config: main |
|
split: test |
|
args: |
|
num_few_shot: 5 |
|
metrics: |
|
- type: acc |
|
value: 0.0 |
|
name: accuracy |
|
source: |
|
url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=MaziyarPanahi/TheTop-5x7B-Instruct-P-v0.1 |
|
name: Open LLM Leaderboard |
|
--- |
|
|
|
Merge of top 7B models with PASS method |
|
|
|
> mergekit is a toolkit for merging pre-trained language models. mergekit uses an out-of-core approach to perform unreasonably elaborate merges in resource-constrained situations. Merges can be run entirely on CPU or accelerated with as little as 8 GB of VRAM. Many merging algorithms are supported, with more coming as they catch my attention. |
|
|
|
## Eval |
|
|
|
```python |
|
{ |
|
"all": { |
|
"acc": 0.6152059168567449, |
|
"acc_stderr": 0.031951119145286845, |
|
"acc_norm": 0.6274010157580394, |
|
"acc_norm_stderr": 0.032831804892806175, |
|
"mc1": 0.25091799265605874, |
|
"mc1_stderr": 0.015176985027707694, |
|
"mc2": 0.5006656333594469, |
|
"mc2_stderr": 0.01636490303268174 |
|
}, |
|
"harness|arc:challenge|25": { |
|
"acc": 0.3447098976109215, |
|
"acc_stderr": 0.013888816286782112, |
|
"acc_norm": 0.3856655290102389, |
|
"acc_norm_stderr": 0.01422425097325717 |
|
}, |
|
"harness|hellaswag|10": { |
|
"acc": 0.34116709818761204, |
|
"acc_stderr": 0.004731324409133264, |
|
"acc_norm": 0.515435172276439, |
|
"acc_norm_stderr": 0.004987403268345035 |
|
}, |
|
"harness|hendrycksTest-abstract_algebra|5": { |
|
"acc": 0.27, |
|
"acc_stderr": 0.04461960433384741, |
|
"acc_norm": 0.27, |
|
"acc_norm_stderr": 0.04461960433384741 |
|
}, |
|
"harness|hendrycksTest-anatomy|5": { |
|
"acc": 0.5703703703703704, |
|
"acc_stderr": 0.042763494943765995, |
|
"acc_norm": 0.5703703703703704, |
|
"acc_norm_stderr": 0.042763494943765995 |
|
}, |
|
"harness|hendrycksTest-astronomy|5": { |
|
"acc": 0.6842105263157895, |
|
"acc_stderr": 0.0378272898086547, |
|
"acc_norm": 0.6842105263157895, |
|
"acc_norm_stderr": 0.0378272898086547 |
|
}, |
|
"harness|hendrycksTest-business_ethics|5": { |
|
"acc": 0.62, |
|
"acc_stderr": 0.048783173121456316, |
|
"acc_norm": 0.62, |
|
"acc_norm_stderr": 0.048783173121456316 |
|
}, |
|
"harness|hendrycksTest-clinical_knowledge|5": { |
|
"acc": 0.7169811320754716, |
|
"acc_stderr": 0.027724236492700918, |
|
"acc_norm": 0.7169811320754716, |
|
"acc_norm_stderr": 0.027724236492700918 |
|
}, |
|
"harness|hendrycksTest-college_biology|5": { |
|
"acc": 0.7638888888888888, |
|
"acc_stderr": 0.03551446610810826, |
|
"acc_norm": 0.7638888888888888, |
|
"acc_norm_stderr": 0.03551446610810826 |
|
}, |
|
"harness|hendrycksTest-college_chemistry|5": { |
|
"acc": 0.45, |
|
"acc_stderr": 0.05, |
|
"acc_norm": 0.45, |
|
"acc_norm_stderr": 0.05 |
|
}, |
|
"harness|hendrycksTest-college_computer_science|5": { |
|
"acc": 0.46, |
|
"acc_stderr": 0.05009082659620333, |
|
"acc_norm": 0.46, |
|
"acc_norm_stderr": 0.05009082659620333 |
|
}, |
|
"harness|hendrycksTest-college_mathematics|5": { |
|
"acc": 0.32, |
|
"acc_stderr": 0.04688261722621504, |
|
"acc_norm": 0.32, |
|
"acc_norm_stderr": 0.04688261722621504 |
|
}, |
|
"harness|hendrycksTest-college_medicine|5": { |
|
"acc": 0.6358381502890174, |
|
"acc_stderr": 0.03669072477416907, |
|
"acc_norm": 0.6358381502890174, |
|
"acc_norm_stderr": 0.03669072477416907 |
|
}, |
|
"harness|hendrycksTest-college_physics|5": { |
|
"acc": 0.4019607843137255, |
|
"acc_stderr": 0.048786087144669955, |
|
"acc_norm": 0.4019607843137255, |
|
"acc_norm_stderr": 0.048786087144669955 |
|
}, |
|
"harness|hendrycksTest-computer_security|5": { |
|
"acc": 0.76, |
|
"acc_stderr": 0.04292346959909283, |
|
"acc_norm": 0.76, |
|
"acc_norm_stderr": 0.04292346959909283 |
|
}, |
|
"harness|hendrycksTest-conceptual_physics|5": { |
|
"acc": 0.5446808510638298, |
|
"acc_stderr": 0.03255525359340355, |
|
"acc_norm": 0.5446808510638298, |
|
"acc_norm_stderr": 0.03255525359340355 |
|
}, |
|
"harness|hendrycksTest-econometrics|5": { |
|
"acc": 0.4824561403508772, |
|
"acc_stderr": 0.04700708033551038, |
|
"acc_norm": 0.4824561403508772, |
|
"acc_norm_stderr": 0.04700708033551038 |
|
}, |
|
"harness|hendrycksTest-electrical_engineering|5": { |
|
"acc": 0.5172413793103449, |
|
"acc_stderr": 0.04164188720169375, |
|
"acc_norm": 0.5172413793103449, |
|
"acc_norm_stderr": 0.04164188720169375 |
|
}, |
|
"harness|hendrycksTest-elementary_mathematics|5": { |
|
"acc": 0.42857142857142855, |
|
"acc_stderr": 0.025487187147859372, |
|
"acc_norm": 0.42857142857142855, |
|
"acc_norm_stderr": 0.025487187147859372 |
|
}, |
|
"harness|hendrycksTest-formal_logic|5": { |
|
"acc": 0.3968253968253968, |
|
"acc_stderr": 0.043758884927270605, |
|
"acc_norm": 0.3968253968253968, |
|
"acc_norm_stderr": 0.043758884927270605 |
|
}, |
|
"harness|hendrycksTest-global_facts|5": { |
|
"acc": 0.34, |
|
"acc_stderr": 0.04760952285695236, |
|
"acc_norm": 0.34, |
|
"acc_norm_stderr": 0.04760952285695236 |
|
}, |
|
"harness|hendrycksTest-high_school_biology|5": { |
|
"acc": 0.7741935483870968, |
|
"acc_stderr": 0.023785577884181015, |
|
"acc_norm": 0.7741935483870968, |
|
"acc_norm_stderr": 0.023785577884181015 |
|
}, |
|
"harness|hendrycksTest-high_school_chemistry|5": { |
|
"acc": 0.5123152709359606, |
|
"acc_stderr": 0.035169204442208966, |
|
"acc_norm": 0.5123152709359606, |
|
"acc_norm_stderr": 0.035169204442208966 |
|
}, |
|
"harness|hendrycksTest-high_school_computer_science|5": { |
|
"acc": 0.66, |
|
"acc_stderr": 0.04760952285695237, |
|
"acc_norm": 0.66, |
|
"acc_norm_stderr": 0.04760952285695237 |
|
}, |
|
"harness|hendrycksTest-high_school_european_history|5": { |
|
"acc": 0.7636363636363637, |
|
"acc_stderr": 0.03317505930009181, |
|
"acc_norm": 0.7636363636363637, |
|
"acc_norm_stderr": 0.03317505930009181 |
|
}, |
|
"harness|hendrycksTest-high_school_geography|5": { |
|
"acc": 0.7373737373737373, |
|
"acc_stderr": 0.03135305009533085, |
|
"acc_norm": 0.7373737373737373, |
|
"acc_norm_stderr": 0.03135305009533085 |
|
}, |
|
"harness|hendrycksTest-high_school_government_and_politics|5": { |
|
"acc": 0.8808290155440415, |
|
"acc_stderr": 0.023381935348121437, |
|
"acc_norm": 0.8808290155440415, |
|
"acc_norm_stderr": 0.023381935348121437 |
|
}, |
|
"harness|hendrycksTest-high_school_macroeconomics|5": { |
|
"acc": 0.617948717948718, |
|
"acc_stderr": 0.024635549163908237, |
|
"acc_norm": 0.617948717948718, |
|
"acc_norm_stderr": 0.024635549163908237 |
|
}, |
|
"harness|hendrycksTest-high_school_mathematics|5": { |
|
"acc": 0.2777777777777778, |
|
"acc_stderr": 0.027309140588230203, |
|
"acc_norm": 0.2777777777777778, |
|
"acc_norm_stderr": 0.027309140588230203 |
|
}, |
|
"harness|hendrycksTest-high_school_microeconomics|5": { |
|
"acc": 0.6512605042016807, |
|
"acc_stderr": 0.030956636328566545, |
|
"acc_norm": 0.6512605042016807, |
|
"acc_norm_stderr": 0.030956636328566545 |
|
}, |
|
"harness|hendrycksTest-high_school_physics|5": { |
|
"acc": 0.32450331125827814, |
|
"acc_stderr": 0.038227469376587525, |
|
"acc_norm": 0.32450331125827814, |
|
"acc_norm_stderr": 0.038227469376587525 |
|
}, |
|
"harness|hendrycksTest-high_school_psychology|5": { |
|
"acc": 0.8440366972477065, |
|
"acc_stderr": 0.015555802713590158, |
|
"acc_norm": 0.8440366972477065, |
|
"acc_norm_stderr": 0.015555802713590158 |
|
}, |
|
"harness|hendrycksTest-high_school_statistics|5": { |
|
"acc": 0.4722222222222222, |
|
"acc_stderr": 0.0340470532865388, |
|
"acc_norm": 0.4722222222222222, |
|
"acc_norm_stderr": 0.0340470532865388 |
|
}, |
|
"harness|hendrycksTest-high_school_us_history|5": { |
|
"acc": 0.8431372549019608, |
|
"acc_stderr": 0.025524722324553346, |
|
"acc_norm": 0.8431372549019608, |
|
"acc_norm_stderr": 0.025524722324553346 |
|
}, |
|
"harness|hendrycksTest-high_school_world_history|5": { |
|
"acc": 0.810126582278481, |
|
"acc_stderr": 0.025530100460233497, |
|
"acc_norm": 0.810126582278481, |
|
"acc_norm_stderr": 0.025530100460233497 |
|
}, |
|
"harness|hendrycksTest-human_aging|5": { |
|
"acc": 0.7174887892376681, |
|
"acc_stderr": 0.03021683101150877, |
|
"acc_norm": 0.7174887892376681, |
|
"acc_norm_stderr": 0.03021683101150877 |
|
}, |
|
"harness|hendrycksTest-human_sexuality|5": { |
|
"acc": 0.7786259541984732, |
|
"acc_stderr": 0.0364129708131373, |
|
"acc_norm": 0.7786259541984732, |
|
"acc_norm_stderr": 0.0364129708131373 |
|
}, |
|
"harness|hendrycksTest-international_law|5": { |
|
"acc": 0.7768595041322314, |
|
"acc_stderr": 0.03800754475228733, |
|
"acc_norm": 0.7768595041322314, |
|
"acc_norm_stderr": 0.03800754475228733 |
|
}, |
|
"harness|hendrycksTest-jurisprudence|5": { |
|
"acc": 0.8148148148148148, |
|
"acc_stderr": 0.03755265865037181, |
|
"acc_norm": 0.8148148148148148, |
|
"acc_norm_stderr": 0.03755265865037181 |
|
}, |
|
"harness|hendrycksTest-logical_fallacies|5": { |
|
"acc": 0.7914110429447853, |
|
"acc_stderr": 0.031921934489347235, |
|
"acc_norm": 0.7914110429447853, |
|
"acc_norm_stderr": 0.031921934489347235 |
|
}, |
|
"harness|hendrycksTest-machine_learning|5": { |
|
"acc": 0.5446428571428571, |
|
"acc_stderr": 0.04726835553719097, |
|
"acc_norm": 0.5446428571428571, |
|
"acc_norm_stderr": 0.04726835553719097 |
|
}, |
|
"harness|hendrycksTest-management|5": { |
|
"acc": 0.8349514563106796, |
|
"acc_stderr": 0.036756688322331886, |
|
"acc_norm": 0.8349514563106796, |
|
"acc_norm_stderr": 0.036756688322331886 |
|
}, |
|
"harness|hendrycksTest-marketing|5": { |
|
"acc": 0.8290598290598291, |
|
"acc_stderr": 0.024662496845209804, |
|
"acc_norm": 0.8290598290598291, |
|
"acc_norm_stderr": 0.024662496845209804 |
|
}, |
|
"harness|hendrycksTest-medical_genetics|5": { |
|
"acc": 0.69, |
|
"acc_stderr": 0.04648231987117316, |
|
"acc_norm": 0.69, |
|
"acc_norm_stderr": 0.04648231987117316 |
|
}, |
|
"harness|hendrycksTest-miscellaneous|5": { |
|
"acc": 0.8250319284802043, |
|
"acc_stderr": 0.013586619219903324, |
|
"acc_norm": 0.8250319284802043, |
|
"acc_norm_stderr": 0.013586619219903324 |
|
}, |
|
"harness|hendrycksTest-moral_disputes|5": { |
|
"acc": 0.7283236994219653, |
|
"acc_stderr": 0.023948512905468348, |
|
"acc_norm": 0.7283236994219653, |
|
"acc_norm_stderr": 0.023948512905468348 |
|
}, |
|
"harness|hendrycksTest-moral_scenarios|5": { |
|
"acc": 0.36312849162011174, |
|
"acc_stderr": 0.016083749986853704, |
|
"acc_norm": 0.36312849162011174, |
|
"acc_norm_stderr": 0.016083749986853704 |
|
}, |
|
"harness|hendrycksTest-nutrition|5": { |
|
"acc": 0.7450980392156863, |
|
"acc_stderr": 0.02495418432487991, |
|
"acc_norm": 0.7450980392156863, |
|
"acc_norm_stderr": 0.02495418432487991 |
|
}, |
|
"harness|hendrycksTest-philosophy|5": { |
|
"acc": 0.7202572347266881, |
|
"acc_stderr": 0.02549425935069491, |
|
"acc_norm": 0.7202572347266881, |
|
"acc_norm_stderr": 0.02549425935069491 |
|
}, |
|
"harness|hendrycksTest-prehistory|5": { |
|
"acc": 0.7530864197530864, |
|
"acc_stderr": 0.023993501709042114, |
|
"acc_norm": 0.7530864197530864, |
|
"acc_norm_stderr": 0.023993501709042114 |
|
}, |
|
"harness|hendrycksTest-professional_accounting|5": { |
|
"acc": 0.4787234042553192, |
|
"acc_stderr": 0.029800481645628693, |
|
"acc_norm": 0.4787234042553192, |
|
"acc_norm_stderr": 0.029800481645628693 |
|
}, |
|
"harness|hendrycksTest-professional_law|5": { |
|
"acc": 0.4367666232073012, |
|
"acc_stderr": 0.01266770191960366, |
|
"acc_norm": 0.4367666232073012, |
|
"acc_norm_stderr": 0.01266770191960366 |
|
}, |
|
"harness|hendrycksTest-professional_medicine|5": { |
|
"acc": 0.6176470588235294, |
|
"acc_stderr": 0.029520095697687765, |
|
"acc_norm": 0.6176470588235294, |
|
"acc_norm_stderr": 0.029520095697687765 |
|
}, |
|
"harness|hendrycksTest-professional_psychology|5": { |
|
"acc": 0.6699346405228758, |
|
"acc_stderr": 0.019023726160724553, |
|
"acc_norm": 0.6699346405228758, |
|
"acc_norm_stderr": 0.019023726160724553 |
|
}, |
|
"harness|hendrycksTest-public_relations|5": { |
|
"acc": 0.6545454545454545, |
|
"acc_stderr": 0.04554619617541054, |
|
"acc_norm": 0.6545454545454545, |
|
"acc_norm_stderr": 0.04554619617541054 |
|
}, |
|
"harness|hendrycksTest-security_studies|5": { |
|
"acc": 0.726530612244898, |
|
"acc_stderr": 0.028535560337128445, |
|
"acc_norm": 0.726530612244898, |
|
"acc_norm_stderr": 0.028535560337128445 |
|
}, |
|
"harness|hendrycksTest-sociology|5": { |
|
"acc": 0.845771144278607, |
|
"acc_stderr": 0.025538433368578334, |
|
"acc_norm": 0.845771144278607, |
|
"acc_norm_stderr": 0.025538433368578334 |
|
}, |
|
"harness|hendrycksTest-us_foreign_policy|5": { |
|
"acc": 0.85, |
|
"acc_stderr": 0.03588702812826371, |
|
"acc_norm": 0.85, |
|
"acc_norm_stderr": 0.03588702812826371 |
|
}, |
|
"harness|hendrycksTest-virology|5": { |
|
"acc": 0.5542168674698795, |
|
"acc_stderr": 0.03869543323472101, |
|
"acc_norm": 0.5542168674698795, |
|
"acc_norm_stderr": 0.03869543323472101 |
|
}, |
|
"harness|hendrycksTest-world_religions|5": { |
|
"acc": 0.8362573099415205, |
|
"acc_stderr": 0.028380919596145866, |
|
"acc_norm": 0.8362573099415205, |
|
"acc_norm_stderr": 0.028380919596145866 |
|
}, |
|
"harness|truthfulqa:mc|0": { |
|
"mc1": 0.25091799265605874, |
|
"mc1_stderr": 0.015176985027707694, |
|
"mc2": 0.5006656333594469, |
|
"mc2_stderr": 0.01636490303268174 |
|
}, |
|
"harness|winogrande|5": { |
|
"acc": 0.7261247040252565, |
|
"acc_stderr": 0.012533292732620296 |
|
}, |
|
"harness|gsm8k|5": { |
|
"acc": 0.0, |
|
"acc_stderr": 0.0 |
|
} |
|
} |
|
|
|
``` |
|
# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) |
|
Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_MaziyarPanahi__TheTop-5x7B-Instruct-P-v0.1) |
|
|
|
| Metric |Value| |
|
|---------------------------------|----:| |
|
|Avg. |46.02| |
|
|AI2 Reasoning Challenge (25-Shot)|38.57| |
|
|HellaSwag (10-Shot) |51.54| |
|
|MMLU (5-Shot) |63.36| |
|
|TruthfulQA (0-shot) |50.07| |
|
|Winogrande (5-shot) |72.61| |
|
|GSM8k (5-shot) | 0.00| |
|
|
|
|
