region:us
Measuring Massive Multitask Language Understanding by Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt (ICLR 2021).
@article{hendryckstest2021,
  title={Measuring Massive Multitask Language Understanding},
  author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
  journal={Proceedings of the International Conference on Learning Representations (ICLR)},
  year={2021}
}
---
dataset_info:
- config_name: high_school_european_history
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 268045
    num_examples: 164
  - name: validation
    num_bytes: 27437
    num_examples: 17
  - name: train
    num_bytes: 9449
    num_examples: 4
  download_size: 166184960
  dataset_size: 304931
- config_name: business_ethics
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 32080
    num_examples: 99
  - name: validation
    num_bytes: 2735
    num_examples: 10
  - name: train
    num_bytes: 1770
    num_examples: 4
  download_size: 166184960
  dataset_size: 36585
- config_name: clinical_knowledge
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 60710
    num_examples: 264
  - name: validation
    num_bytes: 6231
    num_examples: 28
  - name: train
    num_bytes: 1026
    num_examples: 4
  download_size: 166184960
  dataset_size: 67967
- config_name: medical_genetics
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 20021
    num_examples: 99
  - name: validation
    num_bytes: 2590
    num_examples: 10
  - name: train
    num_bytes: 854
    num_examples: 4
  download_size: 166184960
  dataset_size: 23465
- config_name: high_school_us_history
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 294113
    num_examples: 203
  - name: validation
    num_bytes: 30202
    num_examples: 21
  - name: train
    num_bytes: 7341
    num_examples: 4
  download_size: 166184960
  dataset_size: 331656
- config_name: high_school_physics
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 58279
    num_examples: 150
  - name: validation
    num_bytes: 6189
    num_examples: 16
  - name: train
    num_bytes: 1193
    num_examples: 4
  download_size: 166184960
  dataset_size: 65661
- config_name: high_school_world_history
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 376057
    num_examples: 236
  - name: validation
    num_bytes: 44247
    num_examples: 25
  - name: train
    num_bytes: 4339
    num_examples: 4
  download_size: 166184960
  dataset_size: 424643
- config_name: virology
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 37496
    num_examples: 165
  - name: validation
    num_bytes: 5124
    num_examples: 17
  - name: train
    num_bytes: 848
    num_examples: 4
  download_size: 166184960
  dataset_size: 43468
- config_name: high_school_microeconomics
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 73766
    num_examples: 237
  - name: validation
    num_bytes: 7165
    num_examples: 25
  - name: train
    num_bytes: 855
    num_examples: 4
  download_size: 166184960
  dataset_size: 81786
- config_name: econometrics
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 45258
    num_examples: 113
  - name: validation
    num_bytes: 4552
    num_examples: 11
  - name: train
    num_bytes: 1452
    num_examples: 4
  download_size: 166184960
  dataset_size: 51262
- config_name: college_computer_science
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 41567
    num_examples: 99
  - name: validation
    num_bytes: 4157
    num_examples: 10
  - name: train
    num_bytes: 2496
    num_examples: 4
  download_size: 166184960
  dataset_size: 48220
- config_name: high_school_biology
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 107194
    num_examples: 309
  - name: validation
    num_bytes: 10054
    num_examples: 31
  - name: train
    num_bytes: 1481
    num_examples: 4
  download_size: 166184960
  dataset_size: 118729
- config_name: abstract_algebra
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 18504
    num_examples: 99
  - name: validation
    num_bytes: 1852
    num_examples: 10
  - name: train
    num_bytes: 698
    num_examples: 4
  download_size: 166184960
  dataset_size: 21054
- config_name: professional_accounting
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 122155
    num_examples: 281
  - name: validation
    num_bytes: 13749
    num_examples: 30
  - name: train
    num_bytes: 1683
    num_examples: 4
  download_size: 166184960
  dataset_size: 137587
- config_name: philosophy
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 77720
    num_examples: 310
  - name: validation
    num_bytes: 8352
    num_examples: 33
  - name: train
    num_bytes: 698
    num_examples: 4
  download_size: 166184960
  dataset_size: 86770
- config_name: professional_medicine
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 214495
    num_examples: 271
  - name: validation
    num_bytes: 23003
    num_examples: 30
  - name: train
    num_bytes: 2531
    num_examples: 4
  download_size: 166184960
  dataset_size: 240029
- config_name: nutrition
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 90097
    num_examples: 305
  - name: validation
    num_bytes: 7826
    num_examples: 32
  - name: train
    num_bytes: 1455
    num_examples: 4
  download_size: 166184960
  dataset_size: 99378
- config_name: global_facts
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 17571
    num_examples: 99
  - name: validation
    num_bytes: 1646
    num_examples: 9
  - name: train
    num_bytes: 666
    num_examples: 4
  download_size: 166184960
  dataset_size: 19883
- config_name: machine_learning
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 32810
    num_examples: 111
  - name: validation
    num_bytes: 2701
    num_examples: 10
  - name: train
    num_bytes: 1971
    num_examples: 4
  download_size: 166184960
  dataset_size: 37482
- config_name: security_studies
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 202888
    num_examples: 244
  - name: validation
    num_bytes: 21703
    num_examples: 26
  - name: train
    num_bytes: 3889
    num_examples: 4
  download_size: 166184960
  dataset_size: 228480
- config_name: public_relations
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 27797
    num_examples: 109
  - name: validation
    num_bytes: 4180
    num_examples: 11
  - name: train
    num_bytes: 1064
    num_examples: 4
  download_size: 166184960
  dataset_size: 33041
- config_name: professional_psychology
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 221179
    num_examples: 611
  - name: validation
    num_bytes: 28283
    num_examples: 68
  - name: train
    num_bytes: 1348
    num_examples: 4
  download_size: 166184960
  dataset_size: 250810
- config_name: prehistory
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 87125
    num_examples: 323
  - name: validation
    num_bytes: 9915
    num_examples: 34
  - name: train
    num_bytes: 1484
    num_examples: 4
  download_size: 166184960
  dataset_size: 98524
- config_name: anatomy
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 31810
    num_examples: 134
  - name: validation
    num_bytes: 2879
    num_examples: 13
  - name: train
    num_bytes: 717
    num_examples: 4
  download_size: 166184960
  dataset_size: 35406
- config_name: human_sexuality
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 31019
    num_examples: 130
  - name: validation
    num_bytes: 2042
    num_examples: 11
  - name: train
    num_bytes: 861
    num_examples: 4
  download_size: 166184960
  dataset_size: 33922
- config_name: college_medicine
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 80363
    num_examples: 172
  - name: validation
    num_bytes: 7079
    num_examples: 21
  - name: train
    num_bytes: 1434
    num_examples: 4
  download_size: 166184960
  dataset_size: 88876
- config_name: high_school_government_and_politics
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 64098
    num_examples: 192
  - name: validation
    num_bytes: 6317
    num_examples: 20
  - name: train
    num_bytes: 1314
    num_examples: 4
  download_size: 166184960
  dataset_size: 71729
- config_name: college_chemistry
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 23837
    num_examples: 99
  - name: validation
    num_bytes: 2039
    num_examples: 7
  - name: train
    num_bytes: 892
    num_examples: 4
  download_size: 166184960
  dataset_size: 26768
- config_name: logical_fallacies
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 48758
    num_examples: 162
  - name: validation
    num_bytes: 4699
    num_examples: 17
  - name: train
    num_bytes: 1256
    num_examples: 4
  download_size: 166184960
  dataset_size: 54713
- config_name: high_school_geography
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 40424
    num_examples: 197
  - name: validation
    num_bytes: 3876
    num_examples: 21
  - name: train
    num_bytes: 1092
    num_examples: 4
  download_size: 166184960
  dataset_size: 45392
- config_name: elementary_mathematics
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 67369
    num_examples: 377
  - name: validation
    num_bytes: 8458
    num_examples: 40
  - name: train
    num_bytes: 1223
    num_examples: 4
  download_size: 166184960
  dataset_size: 77050
- config_name: human_aging
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 44398
    num_examples: 222
  - name: validation
    num_bytes: 4226
    num_examples: 22
  - name: train
    num_bytes: 774
    num_examples: 4
  download_size: 166184960
  dataset_size: 49398
- config_name: college_mathematics
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 23739
    num_examples: 99
  - name: validation
    num_bytes: 2362
    num_examples: 10
  - name: train
    num_bytes: 1146
    num_examples: 4
  download_size: 166184960
  dataset_size: 27247
- config_name: high_school_psychology
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 155363
    num_examples: 544
  - name: validation
    num_bytes: 16538
    num_examples: 59
  - name: train
    num_bytes: 1618
    num_examples: 4
  download_size: 166184960
  dataset_size: 173519
- config_name: formal_logic
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 47829
    num_examples: 125
  - name: validation
    num_bytes: 5774
    num_examples: 13
  - name: train
    num_bytes: 1568
    num_examples: 4
  download_size: 166184960
  dataset_size: 55171
- config_name: high_school_statistics
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 108742
    num_examples: 215
  - name: validation
    num_bytes: 9537
    num_examples: 22
  - name: train
    num_bytes: 1993
    num_examples: 4
  download_size: 166184960
  dataset_size: 120272
- config_name: international_law
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 52439
    num_examples: 120
  - name: validation
    num_bytes: 5918
    num_examples: 12
  - name: train
    num_bytes: 2017
    num_examples: 4
  download_size: 166184960
  dataset_size: 60374
- config_name: high_school_mathematics
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 52702
    num_examples: 269
  - name: validation
    num_bytes: 5277
    num_examples: 28
  - name: train
    num_bytes: 826
    num_examples: 4
  download_size: 166184960
  dataset_size: 58805
- config_name: high_school_computer_science
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 43696
    num_examples: 99
  - name: validation
    num_bytes: 3088
    num_examples: 8
  - name: train
    num_bytes: 2463
    num_examples: 4
  download_size: 166184960
  dataset_size: 49247
- config_name: conceptual_physics
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 38927
    num_examples: 234
  - name: validation
    num_bytes: 4154
    num_examples: 25
  - name: train
    num_bytes: 728
    num_examples: 4
  download_size: 166184960
  dataset_size: 43809
- config_name: miscellaneous
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 141981
    num_examples: 782
  - name: validation
    num_bytes: 13562
    num_examples: 85
  - name: train
    num_bytes: 565
    num_examples: 4
  download_size: 166184960
  dataset_size: 156108
- config_name: high_school_chemistry
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 56653
    num_examples: 202
  - name: validation
    num_bytes: 6725
    num_examples: 21
  - name: train
    num_bytes: 1074
    num_examples: 4
  download_size: 166184960
  dataset_size: 64452
- config_name: marketing
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 61240
    num_examples: 233
  - name: validation
    num_bytes: 7027
    num_examples: 24
  - name: train
    num_bytes: 1251
    num_examples: 4
  download_size: 166184960
  dataset_size: 69518
- config_name: professional_law
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 1879598
    num_examples: 1533
  - name: validation
    num_bytes: 201226
    num_examples: 169
  - name: train
    num_bytes: 5085
    num_examples: 4
  download_size: 166184960
  dataset_size: 2085909
- config_name: management
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 19112
    num_examples: 102
  - name: validation
    num_bytes: 1598
    num_examples: 10
  - name: train
    num_bytes: 600
    num_examples: 4
  download_size: 166184960
  dataset_size: 21310
- config_name: college_physics
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 29056
    num_examples: 101
  - name: validation
    num_bytes: 2958
    num_examples: 10
  - name: train
    num_bytes: 1164
    num_examples: 4
  download_size: 166184960
  dataset_size: 33178
- config_name: jurisprudence
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 32839
    num_examples: 107
  - name: validation
    num_bytes: 3438
    num_examples: 10
  - name: train
    num_bytes: 929
    num_examples: 4
  download_size: 166184960
  dataset_size: 37206
- config_name: world_religions
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 23974
    num_examples: 170
  - name: validation
    num_bytes: 2504
    num_examples: 18
  - name: train
    num_bytes: 508
    num_examples: 4
  download_size: 166184960
  dataset_size: 26986
- config_name: sociology
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 64513
    num_examples: 200
  - name: validation
    num_bytes: 6818
    num_examples: 21
  - name: train
    num_bytes: 1376
    num_examples: 4
  download_size: 166184960
  dataset_size: 72707
- config_name: us_foreign_policy
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 27391
    num_examples: 99
  - name: validation
    num_bytes: 2729
    num_examples: 10
  - name: train
    num_bytes: 1216
    num_examples: 4
  download_size: 166184960
  dataset_size: 31336
- config_name: high_school_macroeconomics
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 114578
    num_examples: 389
  - name: validation
    num_bytes: 12446
    num_examples: 42
  - name: train
    num_bytes: 927
    num_examples: 4
  download_size: 166184960
  dataset_size: 127951
- config_name: computer_security
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 26220
    num_examples: 99
  - name: validation
    num_bytes: 4178
    num_examples: 10
  - name: train
    num_bytes: 968
    num_examples: 4
  download_size: 166184960
  dataset_size: 31366
- config_name: moral_scenarios
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 367352
    num_examples: 894
  - name: validation
    num_bytes: 41184
    num_examples: 99
  - name: train
    num_bytes: 1598
    num_examples: 4
  download_size: 166184960
  dataset_size: 410134
- config_name: moral_disputes
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 105240
    num_examples: 345
  - name: validation
    num_bytes: 11732
    num_examples: 37
  - name: train
    num_bytes: 1196
    num_examples: 4
  download_size: 166184960
  dataset_size: 118168
- config_name: electrical_engineering
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 23901
    num_examples: 144
  - name: validation
    num_bytes: 2576
    num_examples: 15
  - name: train
    num_bytes: 801
    num_examples: 4
  download_size: 166184960
  dataset_size: 27278
- config_name: astronomy
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 45470
    num_examples: 151
  - name: validation
    num_bytes: 4482
    num_examples: 15
  - name: train
    num_bytes: 1672
    num_examples: 4
  download_size: 166184960
  dataset_size: 51624
- config_name: college_biology
  features:
  - name: input
    dtype: string
  - name: A
    dtype: string
  - name: B
    dtype: string
  - name: C
    dtype: string
  - name: D
    dtype: string
  - name: target
    dtype: string
  splits:
  - name: test
    num_bytes: 47319
    num_examples: 143
  - name: validation
    num_bytes: 4462
    num_examples: 15
  - name: train
    num_bytes: 1103
    num_examples: 4
  download_size: 166184960
  dataset_size: 52884
---
# MMLU dataset
Measuring Massive Multitask Language Understanding: https://github.com/hendrycks/test

```
@article{hendryckstest2021,
  title={Measuring Massive Multitask Language Understanding},
  author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
  journal={Proceedings of the International Conference on Learning Representations (ICLR)},
  year={2021}
}
```