task_categories:question-answering task_categories:multiple-choice task_ids:multiple-choice-qa size_categories:10K<n<100K language:ru language:en arxiv:2009.03300 region:us
---
pretty_name: MMLU RU/EN
language:
- ru
- en
size_categories:
- 10K<n<100K
task_categories:
- question-answering
- multiple-choice
task_ids:
  - multiple-choice-qa
dataset_info:
- config_name: abstract_algebra
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 2182
    num_examples: 5
  - name: val
    num_bytes: 5220
    num_examples: 11
  - name: test
    num_bytes: 50926
    num_examples: 100
  download_size: 5548198
  dataset_size: 58328
- config_name: anatomy
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 2482
    num_examples: 5
  - name: val
    num_bytes: 8448
    num_examples: 14
  - name: test
    num_bytes: 91387
    num_examples: 135
  download_size: 5548198
  dataset_size: 102317
- config_name: astronomy
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 6049
    num_examples: 5
  - name: val
    num_bytes: 14187
    num_examples: 16
  - name: test
    num_bytes: 130167
    num_examples: 152
  download_size: 5548198
  dataset_size: 150403
- config_name: business_ethics
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 6197
    num_examples: 5
  - name: val
    num_bytes: 8963
    num_examples: 11
  - name: test
    num_bytes: 96566
    num_examples: 100
  download_size: 5548198
  dataset_size: 111726
- config_name: clinical_knowledge
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3236
    num_examples: 5
  - name: val
    num_bytes: 18684
    num_examples: 29
  - name: test
    num_bytes: 178043
    num_examples: 265
  download_size: 5548198
  dataset_size: 199963
- config_name: college_biology
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 4232
    num_examples: 5
  - name: val
    num_bytes: 13521
    num_examples: 16
  - name: test
    num_bytes: 139322
    num_examples: 144
  download_size: 5548198
  dataset_size: 157075
- config_name: college_chemistry
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3533
    num_examples: 5
  - name: val
    num_bytes: 6157
    num_examples: 8
  - name: test
    num_bytes: 65540
    num_examples: 100
  download_size: 5548198
  dataset_size: 75230
- config_name: college_computer_science
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 7513
    num_examples: 5
  - name: val
    num_bytes: 13341
    num_examples: 11
  - name: test
    num_bytes: 120578
    num_examples: 100
  download_size: 5548198
  dataset_size: 141432
- config_name: college_mathematics
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3841
    num_examples: 5
  - name: val
    num_bytes: 6835
    num_examples: 11
  - name: test
    num_bytes: 65110
    num_examples: 100
  download_size: 5548198
  dataset_size: 75786
- config_name: college_medicine
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 4659
    num_examples: 5
  - name: val
    num_bytes: 22116
    num_examples: 22
  - name: test
    num_bytes: 235856
    num_examples: 173
  download_size: 5548198
  dataset_size: 262631
- config_name: college_physics
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3740
    num_examples: 5
  - name: val
    num_bytes: 9491
    num_examples: 11
  - name: test
    num_bytes: 81480
    num_examples: 102
  download_size: 5548198
  dataset_size: 94711
- config_name: computer_security
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3150
    num_examples: 5
  - name: val
    num_bytes: 12859
    num_examples: 11
  - name: test
    num_bytes: 77969
    num_examples: 100
  download_size: 5548198
  dataset_size: 93978
- config_name: conceptual_physics
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 2611
    num_examples: 5
  - name: val
    num_bytes: 12480
    num_examples: 26
  - name: test
    num_bytes: 112243
    num_examples: 235
  download_size: 5548198
  dataset_size: 127334
- config_name: econometrics
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 4548
    num_examples: 5
  - name: val
    num_bytes: 13874
    num_examples: 12
  - name: test
    num_bytes: 128633
    num_examples: 114
  download_size: 5548198
  dataset_size: 147055
- config_name: electrical_engineering
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 2598
    num_examples: 5
  - name: val
    num_bytes: 8003
    num_examples: 16
  - name: test
    num_bytes: 70846
    num_examples: 145
  download_size: 5548198
  dataset_size: 81447
- config_name: elementary_mathematics
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3760
    num_examples: 5
  - name: val
    num_bytes: 23416
    num_examples: 41
  - name: test
    num_bytes: 181090
    num_examples: 378
  download_size: 5548198
  dataset_size: 208266
- config_name: formal_logic
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 4715
    num_examples: 5
  - name: val
    num_bytes: 17099
    num_examples: 14
  - name: test
    num_bytes: 133930
    num_examples: 126
  download_size: 5548198
  dataset_size: 155744
- config_name: global_facts
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3450
    num_examples: 5
  - name: val
    num_bytes: 4971
    num_examples: 10
  - name: test
    num_bytes: 51481
    num_examples: 100
  download_size: 5548198
  dataset_size: 59902
- config_name: high_school_biology
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 4759
    num_examples: 5
  - name: val
    num_bytes: 30807
    num_examples: 32
  - name: test
    num_bytes: 310356
    num_examples: 310
  download_size: 5548198
  dataset_size: 345922
- config_name: high_school_chemistry
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3204
    num_examples: 5
  - name: val
    num_bytes: 18948
    num_examples: 22
  - name: test
    num_bytes: 158246
    num_examples: 203
  download_size: 5548198
  dataset_size: 180398
- config_name: high_school_computer_science
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 7933
    num_examples: 5
  - name: val
    num_bytes: 9612
    num_examples: 9
  - name: test
    num_bytes: 126403
    num_examples: 100
  download_size: 5548198
  dataset_size: 143948
- config_name: high_school_european_history
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 32447
    num_examples: 5
  - name: val
    num_bytes: 83098
    num_examples: 18
  - name: test
    num_bytes: 754136
    num_examples: 165
  download_size: 5548198
  dataset_size: 869681
- config_name: high_school_geography
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 4131
    num_examples: 5
  - name: val
    num_bytes: 12467
    num_examples: 22
  - name: test
    num_bytes: 119021
    num_examples: 198
  download_size: 5548198
  dataset_size: 135619
- config_name: high_school_government_and_politics
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 5188
    num_examples: 5
  - name: val
    num_bytes: 20564
    num_examples: 21
  - name: test
    num_bytes: 194050
    num_examples: 193
  download_size: 5548198
  dataset_size: 219802
- config_name: high_school_macroeconomics
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3942
    num_examples: 5
  - name: val
    num_bytes: 37243
    num_examples: 43
  - name: test
    num_bytes: 340699
    num_examples: 390
  download_size: 5548198
  dataset_size: 381884
- config_name: high_school_mathematics
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3244
    num_examples: 5
  - name: val
    num_bytes: 14758
    num_examples: 29
  - name: test
    num_bytes: 140257
    num_examples: 270
  download_size: 5548198
  dataset_size: 158259
- config_name: high_school_microeconomics
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3503
    num_examples: 5
  - name: val
    num_bytes: 22212
    num_examples: 26
  - name: test
    num_bytes: 219097
    num_examples: 238
  download_size: 5548198
  dataset_size: 244812
- config_name: high_school_physics
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3905
    num_examples: 5
  - name: val
    num_bytes: 18535
    num_examples: 17
  - name: test
    num_bytes: 162917
    num_examples: 151
  download_size: 5548198
  dataset_size: 185357
- config_name: high_school_psychology
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 5207
    num_examples: 5
  - name: val
    num_bytes: 49277
    num_examples: 60
  - name: test
    num_bytes: 455603
    num_examples: 545
  download_size: 5548198
  dataset_size: 510087
- config_name: high_school_statistics
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 6823
    num_examples: 5
  - name: val
    num_bytes: 28020
    num_examples: 23
  - name: test
    num_bytes: 312578
    num_examples: 216
  download_size: 5548198
  dataset_size: 347421
- config_name: high_school_us_history
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 25578
    num_examples: 5
  - name: val
    num_bytes: 91278
    num_examples: 22
  - name: test
    num_bytes: 842680
    num_examples: 204
  download_size: 5548198
  dataset_size: 959536
- config_name: high_school_world_history
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 13893
    num_examples: 5
  - name: val
    num_bytes: 129121
    num_examples: 26
  - name: test
    num_bytes: 1068018
    num_examples: 237
  download_size: 5548198
  dataset_size: 1211032
- config_name: human_aging
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 2820
    num_examples: 5
  - name: val
    num_bytes: 13442
    num_examples: 23
  - name: test
    num_bytes: 132242
    num_examples: 223
  download_size: 5548198
  dataset_size: 148504
- config_name: human_sexuality
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3072
    num_examples: 5
  - name: val
    num_bytes: 6699
    num_examples: 12
  - name: test
    num_bytes: 90007
    num_examples: 131
  download_size: 5548198
  dataset_size: 99778
- config_name: international_law
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 6880
    num_examples: 5
  - name: val
    num_bytes: 19166
    num_examples: 13
  - name: test
    num_bytes: 157259
    num_examples: 121
  download_size: 5548198
  dataset_size: 183305
- config_name: jurisprudence
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3568
    num_examples: 5
  - name: val
    num_bytes: 10638
    num_examples: 11
  - name: test
    num_bytes: 97121
    num_examples: 108
  download_size: 5548198
  dataset_size: 111327
- config_name: logical_fallacies
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 4526
    num_examples: 5
  - name: val
    num_bytes: 14547
    num_examples: 18
  - name: test
    num_bytes: 144501
    num_examples: 163
  download_size: 5548198
  dataset_size: 163574
- config_name: machine_learning
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 6966
    num_examples: 5
  - name: val
    num_bytes: 8986
    num_examples: 11
  - name: test
    num_bytes: 95571
    num_examples: 112
  download_size: 5548198
  dataset_size: 111523
- config_name: management
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 2427
    num_examples: 5
  - name: val
    num_bytes: 5210
    num_examples: 11
  - name: test
    num_bytes: 57201
    num_examples: 103
  download_size: 5548198
  dataset_size: 64838
- config_name: marketing
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 4514
    num_examples: 5
  - name: val
    num_bytes: 20832
    num_examples: 25
  - name: test
    num_bytes: 181786
    num_examples: 234
  download_size: 5548198
  dataset_size: 207132
- config_name: medical_genetics
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3226
    num_examples: 5
  - name: val
    num_bytes: 8214
    num_examples: 11
  - name: test
    num_bytes: 57064
    num_examples: 100
  download_size: 5548198
  dataset_size: 68504
- config_name: miscellaneous
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 1782
    num_examples: 5
  - name: val
    num_bytes: 39225
    num_examples: 86
  - name: test
    num_bytes: 407209
    num_examples: 783
  download_size: 5548198
  dataset_size: 448216
- config_name: moral_disputes
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 4910
    num_examples: 5
  - name: val
    num_bytes: 36026
    num_examples: 38
  - name: test
    num_bytes: 313611
    num_examples: 346
  download_size: 5548198
  dataset_size: 354547
- config_name: moral_scenarios
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 6175
    num_examples: 5
  - name: val
    num_bytes: 129062
    num_examples: 100
  - name: test
    num_bytes: 1137631
    num_examples: 895
  download_size: 5548198
  dataset_size: 1272868
- config_name: nutrition
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 6030
    num_examples: 5
  - name: val
    num_bytes: 24210
    num_examples: 33
  - name: test
    num_bytes: 266173
    num_examples: 306
  download_size: 5548198
  dataset_size: 296413
- config_name: philosophy
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 2631
    num_examples: 5
  - name: val
    num_bytes: 25751
    num_examples: 34
  - name: test
    num_bytes: 227086
    num_examples: 311
  download_size: 5548198
  dataset_size: 255468
- config_name: prehistory
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 5394
    num_examples: 5
  - name: val
    num_bytes: 28687
    num_examples: 35
  - name: test
    num_bytes: 251723
    num_examples: 324
  download_size: 5548198
  dataset_size: 285804
- config_name: professional_accounting
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 6277
    num_examples: 5
  - name: val
    num_bytes: 40914
    num_examples: 31
  - name: test
    num_bytes: 364528
    num_examples: 282
  download_size: 5548198
  dataset_size: 411719
- config_name: professional_law
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 19120
    num_examples: 5
  - name: val
    num_bytes: 589307
    num_examples: 170
  - name: test
    num_bytes: 5479411
    num_examples: 1534
  download_size: 5548198
  dataset_size: 6087838
- config_name: professional_medicine
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 10901
    num_examples: 5
  - name: val
    num_bytes: 69703
    num_examples: 31
  - name: test
    num_bytes: 633483
    num_examples: 272
  download_size: 5548198
  dataset_size: 714087
- config_name: professional_psychology
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 6430
    num_examples: 5
  - name: val
    num_bytes: 82745
    num_examples: 69
  - name: test
    num_bytes: 648634
    num_examples: 612
  download_size: 5548198
  dataset_size: 737809
- config_name: public_relations
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 4384
    num_examples: 5
  - name: val
    num_bytes: 13108
    num_examples: 12
  - name: test
    num_bytes: 82403
    num_examples: 110
  download_size: 5548198
  dataset_size: 99895
- config_name: security_studies
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 16064
    num_examples: 5
  - name: val
    num_bytes: 67877
    num_examples: 27
  - name: test
    num_bytes: 611059
    num_examples: 245
  download_size: 5548198
  dataset_size: 695000
- config_name: sociology
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 4693
    num_examples: 5
  - name: val
    num_bytes: 20654
    num_examples: 22
  - name: test
    num_bytes: 191420
    num_examples: 201
  download_size: 5548198
  dataset_size: 216767
- config_name: us_foreign_policy
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 4781
    num_examples: 5
  - name: val
    num_bytes: 9171
    num_examples: 11
  - name: test
    num_bytes: 81649
    num_examples: 100
  download_size: 5548198
  dataset_size: 95601
- config_name: virology
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 3063
    num_examples: 5
  - name: val
    num_bytes: 15618
    num_examples: 18
  - name: test
    num_bytes: 111027
    num_examples: 166
  download_size: 5548198
  dataset_size: 129708
- config_name: world_religions
  features:
  - name: question_en
    dtype: string
  - name: choices_en
    sequence: string
  - name: answer
    dtype:
      class_label:
        names:
          '0': A
          '1': B
          '2': C
          '3': D
  - name: question_ru
    dtype: string
  - name: choices_ru
    sequence: string
  splits:
  - name: dev
    num_bytes: 1691
    num_examples: 5
  - name: val
    num_bytes: 7052
    num_examples: 19
  - name: test
    num_bytes: 65559
    num_examples: 171
  download_size: 5548198
  dataset_size: 74302
---

# MMLU in Russian (Massive Multitask Language Understanding)


## Overview of the Dataset

MMLU dataset for EN/RU, without auxiliary train.
The dataset contains `dev`/`val`/`test` splits for both, English and Russian languages.
Note it doesn't include `auxiliary_train` split, which wasn't translated.
Totally the dataset has ~16k samples per language: 285 `dev`, 1531 `val`, 14042 `test`.


## Description of original MMLU

MMLU dataset covers 57 different tasks.
Each task requires to choose the right answer out of four options for a given question.
Paper "Measuring Massive Multitask Language Understanding": https://arxiv.org/abs/2009.03300v3.
It is also known as the "hendrycks_test".


## Dataset Creation

The translation was made via Yandex.Translate API.
There are some translation mistakes, especially observed with terms and formulas, no fixes were applied.
Initial dataset was taken from: https://people.eecs.berkeley.edu/~hendrycks/data.tar.


## Sample example

```
{
    "question_en": "Why doesn't Venus have seasons like Mars and Earth do?",
    "choices_en": [
        "Its rotation axis is nearly perpendicular to the plane of the Solar System.",
        "It does not have an ozone layer.",
        "It does not rotate fast enough.",
        "It is too close to the Sun."
    ],
    "answer": 0,
    "question_ru": "Почему на Венере нет времен года, как на Марсе и Земле?",
    "choices_ru": [
        "Ось его вращения почти перпендикулярна плоскости Солнечной системы.",
        "У него нет озонового слоя.",
        "Он вращается недостаточно быстро.",
        "Это слишком близко к Солнцу."
    ]
}
```

## Usage

To merge all subsets into dataframe per split:

```python
from collections import defaultdict

import datasets
import pandas as pd


subjects = ["abstract_algebra", "anatomy", "astronomy", "business_ethics", "clinical_knowledge", "college_biology", "college_chemistry", "college_computer_science", "college_mathematics", "college_medicine", "college_physics", "computer_security", "conceptual_physics", "econometrics", "electrical_engineering", "elementary_mathematics", "formal_logic", "global_facts", "high_school_biology", "high_school_chemistry", "high_school_computer_science", "high_school_european_history", "high_school_geography", "high_school_government_and_politics", "high_school_macroeconomics", "high_school_mathematics", "high_school_microeconomics", "high_school_physics", "high_school_psychology", "high_school_statistics", "high_school_us_history", "high_school_world_history", "human_aging", "human_sexuality", "international_law", "jurisprudence", "logical_fallacies", "machine_learning", "management", "marketing", "medical_genetics", "miscellaneous", "moral_disputes", "moral_scenarios", "nutrition", "philosophy", "prehistory", "professional_accounting", "professional_law", "professional_medicine", "professional_psychology", "public_relations", "security_studies", "sociology", "us_foreign_policy", "virology", "world_religions"]

splits = ["dev", "val", "test"]

all_datasets = {x: datasets.load_dataset("NLPCoreTeam/mmlu_ru", name=x) for x in subjects}

res = defaultdict(list)
for subject in subjects:
    for split in splits:
        dataset = all_datasets[subject][split]
        df = dataset.to_pandas()
        int2str = dataset.features['answer'].int2str
        df['answer'] = df['answer'].map(int2str)
        df.insert(loc=0, column='subject_en', value=subject)
        res[split].append(df)

res = {k: pd.concat(v) for k, v in res.items()}

df_dev = res['dev']
df_val = res['val']
df_test = res['test']
```


## Evaluation

This dataset is intended to evaluate LLMs with few-shot/zero-shot setup.

Evaluation code: https://github.com/NLP-Core-Team/mmlu_ru

Also resources might be helpful:
1. https://github.com/hendrycks/test
1. https://github.com/openai/evals/blob/main/examples/mmlu.ipynb
1. https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/hendrycks_test.py


## Contributions

Dataset added by NLP core team RnD [Telegram channel](https://t.me/nlpcoreteam)