license:mit region:us
This Language Identification Dataset provides a multi-domain corpus in European and Brazilian Portuguese. 
The repository is an anonymyzed version to support a submsission to the EACL 2024 conference.
Further information about the dataset can be soon found in the paper: Enhancing Portuguese Variants Identification with Domain-Agnostic Ensemble Approaches
"""

_DESCRIPTION =
---
license: mit
dataset_info:
- config_name: law
  features:
  - name: text
    dtype: string
  - name: label
    dtype:
      class_label:
        names:
          '0': pt-PT
          '1': pt-BR
  splits:
  - name: train
    num_bytes: 123139395
    num_examples: 397405
  - name: validation
    num_bytes: 56663
    num_examples: 200
  - name: test
    num_bytes: 271938
    num_examples: 1000
  download_size: 163260550
  dataset_size: 123467996
- config_name: literature
  features:
  - name: text
    dtype: string
  - name: label
    dtype:
      class_label:
        names:
          '0': pt-PT
          '1': pt-BR
  splits:
  - name: train
    num_bytes: 3517766
    num_examples: 10315
  - name: validation
    num_bytes: 1484637
    num_examples: 4422
  - name: test
    num_bytes: 2125655
    num_examples: 6317
  download_size: 9807283
  dataset_size: 7128058
- config_name: news
  features:
  - name: text
    dtype: string
  - name: label
    dtype:
      class_label:
        names:
          '0': pt-PT
          '1': pt-BR
  splits:
  - name: train
    num_bytes: 641932369
    num_examples: 905849
  - name: validation
    num_bytes: 275126414
    num_examples: 388222
  - name: test
    num_bytes: 393251206
    num_examples: 554603
  download_size: 1816074839
  dataset_size: 1310309989
- config_name: politics
  features:
  - name: text
    dtype: string
  - name: label
    dtype:
      class_label:
        names:
          '0': pt-PT
          '1': pt-BR
  splits:
  - name: train
    num_bytes: 189377492
    num_examples: 5582
  - name: validation
    num_bytes: 72061567
    num_examples: 2393
  - name: test
    num_bytes: 103743651
    num_examples: 3419
  download_size: 479587978
  dataset_size: 365182710
- config_name: social_media
  features:
  - name: text
    dtype: string
  - name: label
    dtype:
      class_label:
        names:
          '0': pt-PT
          '1': pt-BR
  splits:
  - name: train
    num_bytes: 253714655
    num_examples: 2019173
  - name: validation
    num_bytes: 43449
    num_examples: 400
  - name: test
    num_bytes: 65642
    num_examples: 600
  download_size: 345744861
  dataset_size: 253823746
- config_name: web
  features:
  - name: text
    dtype: string
  - name: label
    dtype:
      class_label:
        names:
          '0': pt-PT
          '1': pt-BR
  splits:
  - name: train
    num_bytes: 59528182
    num_examples: 14480
  - name: validation
    num_bytes: 26157753
    num_examples: 6206
  - name: test
    num_bytes: 35875967
    num_examples: 8866
  download_size: 164136819
  dataset_size: 121561902
---
