arxiv:2306.05179 region:us
---
dataset_info:
- config_name: afrikaans
  features:
  - name: question_text
    dtype: string
  - name: background
    dtype: string
  - name: answer_text
    dtype: string
  - name: options
    sequence: string
  - name: language
    dtype: string
  - name: level
    dtype: string
  - name: subject
    dtype: string
  - name: subject_category
    dtype: string
  splits:
  - name: dev
    num_bytes: 8860
    num_examples: 25
  - name: test
    num_bytes: 194333
    num_examples: 258
  download_size: 71295
  dataset_size: 203193
- config_name: chinese
  features:
  - name: question_text
    dtype: string
  - name: background
    dtype: string
  - name: answer_text
    dtype: string
  - name: options
    sequence: string
  - name: language
    dtype: string
  - name: level
    dtype: string
  - name: subject
    dtype: string
  - name: subject_category
    dtype: string
  splits:
  - name: dev
    num_bytes: 25055
    num_examples: 29
  - name: test
    num_bytes: 485093
    num_examples: 682
  download_size: 289255
  dataset_size: 510148
- config_name: english
  features:
  - name: question_text
    dtype: string
  - name: background
    dtype: string
  - name: answer_text
    dtype: string
  - name: options
    sequence: string
  - name: language
    dtype: string
  - name: level
    dtype: string
  - name: subject
    dtype: string
  - name: subject_category
    dtype: string
  splits:
  - name: dev
    num_bytes: 12792
    num_examples: 32
  - name: test
    num_bytes: 2573796
    num_examples: 1911
  download_size: 697219
  dataset_size: 2586588
- config_name: italian
  features:
  - name: question_text
    dtype: string
  - name: background
    dtype: string
  - name: answer_text
    dtype: string
  - name: options
    sequence: string
  - name: language
    dtype: string
  - name: level
    dtype: string
  - name: subject
    dtype: string
  - name: subject_category
    dtype: string
  splits:
  - name: dev
    num_bytes: 5834
    num_examples: 18
  - name: test
    num_bytes: 2397963
    num_examples: 811
  download_size: 326671
  dataset_size: 2403797
- config_name: javanese
  features:
  - name: question_text
    dtype: string
  - name: background
    dtype: string
  - name: answer_text
    dtype: string
  - name: options
    sequence: string
  - name: language
    dtype: string
  - name: level
    dtype: string
  - name: subject
    dtype: string
  - name: subject_category
    dtype: string
  splits:
  - name: dev
    num_bytes: 1425
    num_examples: 6
  - name: test
    num_bytes: 187280
    num_examples: 371
  download_size: 84085
  dataset_size: 188705
- config_name: portuguese
  features:
  - name: question_text
    dtype: string
  - name: background
    dtype: string
  - name: answer_text
    dtype: string
  - name: options
    sequence: string
  - name: language
    dtype: string
  - name: level
    dtype: string
  - name: subject
    dtype: string
  - name: subject_category
    dtype: string
  splits:
  - name: dev
    num_bytes: 20979
    num_examples: 24
  - name: test
    num_bytes: 941655
    num_examples: 889
  download_size: 614816
  dataset_size: 962634
- config_name: swahili
  features:
  - name: question_text
    dtype: string
  - name: background
    dtype: string
  - name: answer_text
    dtype: string
  - name: options
    sequence: string
  - name: language
    dtype: string
  - name: level
    dtype: string
  - name: subject
    dtype: string
  - name: subject_category
    dtype: string
  splits:
  - name: dev
    num_bytes: 2053
    num_examples: 6
  - name: test
    num_bytes: 607215
    num_examples: 428
  download_size: 94031
  dataset_size: 609268
- config_name: thai
  features:
  - name: question_text
    dtype: string
  - name: background
    dtype: string
  - name: answer_text
    dtype: string
  - name: options
    sequence: string
  - name: language
    dtype: string
  - name: level
    dtype: string
  - name: subject
    dtype: string
  - name: subject_category
    dtype: string
  splits:
  - name: dev
    num_bytes: 16185
    num_examples: 26
  - name: test
    num_bytes: 2249737
    num_examples: 2168
  download_size: 901256
  dataset_size: 2265922
- config_name: vietnamese
  features:
  - name: question_text
    dtype: string
  - name: background
    dtype: string
  - name: answer_text
    dtype: string
  - name: options
    sequence: string
  - name: language
    dtype: string
  - name: level
    dtype: string
  - name: subject
    dtype: string
  - name: subject_category
    dtype: string
  splits:
  - name: dev
    num_bytes: 7974
    num_examples: 28
  - name: test
    num_bytes: 767759
    num_examples: 1789
  download_size: 375774
  dataset_size: 775733
configs:
- config_name: afrikaans
  data_files:
  - split: dev
    path: afrikaans/dev-*
  - split: test
    path: afrikaans/test-*
- config_name: chinese
  data_files:
  - split: dev
    path: chinese/dev-*
  - split: test
    path: chinese/test-*
- config_name: english
  data_files:
  - split: dev
    path: english/dev-*
  - split: test
    path: english/test-*
- config_name: italian
  data_files:
  - split: dev
    path: italian/dev-*
  - split: test
    path: italian/test-*
- config_name: javanese
  data_files:
  - split: dev
    path: javanese/dev-*
  - split: test
    path: javanese/test-*
- config_name: portuguese
  data_files:
  - split: dev
    path: portuguese/dev-*
  - split: test
    path: portuguese/test-*
- config_name: swahili
  data_files:
  - split: dev
    path: swahili/dev-*
  - split: test
    path: swahili/test-*
- config_name: thai
  data_files:
  - split: dev
    path: thai/dev-*
  - split: test
    path: thai/test-*
- config_name: vietnamese
  data_files:
  - split: dev
    path: vietnamese/dev-*
  - split: test
    path: vietnamese/test-*
---

# M3Exam: A Multilingual 🌏, Multimodal 🖼, Multilevel 📈 Benchmark for LLMs

This is the repository for [M3Exam: A Multilingual, Multimodal, Multilevel Benchmark for Examining Large Language Models](https://arxiv.org/abs/2306.05179/).

TL;DR: We introduce M3Exam, a novel benchmark sourced from real and official human exam questions for evaluating LLMs in a multilingual, multimodal, and multilevel context.

![image](https://github.com/DAMO-NLP-SG/M3Exam/blob/main/images/m3exam-examples.jpg?raw=true)
