|
|
|
|
|
|
|
|
|
|
|
|
|
seed: 0 |
|
|
|
|
|
validation_ratio: 0.005 |
|
validation_min_total_docs: 1_000 |
|
validation_max_split_docs: 1_000 |
|
validation_min_split_docs: 10 |
|
tokenizer_train_ratio: 0.1 |
|
|
|
selected_source_ids: |
|
- starcoder |
|
|
|
selected_dataset_ids: |
|
|
|
- pes2o |
|
- math_amps |
|
- eurlex_en |
|
- wikipedia_20231101_en |
|
- wikibooks_en |
|
- wikiquote_en |
|
- wikinews_en |
|
- wikisource_en |
|
- wikivoyage_en |
|
- colossal_oscar_2015-14_en |
|
- colossal_oscar_2016-40_en |
|
- colossal_oscar_2017-43_en |
|
- colossal_oscar_2018-47_en |
|
- colossal_oscar_2019-22_en |
|
- colossal_oscar_2020-24_en |
|
- colossal_oscar_2020-45_en |
|
- colossal_oscar_2021-49_en |
|
- colossal_oscar_2022-27_en |
|
- colossal_oscar_2022-49_en |
|
- colossal_oscar_2023-14_en |
|
- colossal_oscar_2023-23_en |
|
- pile_of_law_r_legaladvice |
|
- pile_of_law_atticus_contracts |
|
- pile_of_law_un_debates |
|
- proof_pile2_open_web_math |
|
- parlamint_gb |
|
- redpajama_stackexchange |
|
|
|
|
|
- cabernet |
|
- eurlex_fr |
|
- legal_mc4_fr |
|
- wikipedia_20231101_fr |
|
- wikibooks_fr |
|
- wikiquote_fr |
|
- wikinews_fr |
|
- wikisource_fr |
|
- wikivoyage_fr |
|
- colossal_oscar_2015-14_fr |
|
- colossal_oscar_2016-40_fr |
|
- colossal_oscar_2017-43_fr |
|
- colossal_oscar_2018-47_fr |
|
- colossal_oscar_2019-22_fr |
|
- colossal_oscar_2020-24_fr |
|
- colossal_oscar_2020-45_fr |
|
- colossal_oscar_2021-49_fr |
|
- colossal_oscar_2022-27_fr |
|
- colossal_oscar_2022-49_fr |
|
- colossal_oscar_2023-14_fr |
|
- colossal_oscar_2023-23_fr |
|
- opensubtitles_fr |
|
- parlamint_fr |
|
|
|
|
|
- spanish_legal |
|
- eurlex_es |
|
- legal_mc4_es |
|
- wikipedia_20231101_es |
|
- wikibooks_es |
|
- wikiquote_es |
|
- wikinews_es |
|
- wikisource_es |
|
- wikivoyage_es |
|
- colossal_oscar_2015-14_es |
|
- colossal_oscar_2016-40_es |
|
- colossal_oscar_2017-43_es |
|
- colossal_oscar_2018-47_es |
|
- colossal_oscar_2019-22_es |
|
- colossal_oscar_2020-24_es |
|
- colossal_oscar_2020-45_es |
|
- colossal_oscar_2021-49_es |
|
- colossal_oscar_2022-27_es |
|
- colossal_oscar_2022-49_es |
|
- colossal_oscar_2023-14_es |
|
- colossal_oscar_2023-23_es |
|
- opensubtitles_es |
|
- parlamint_es |
|
|
|
|
|
- openlegaldata |
|
- dewac |
|
- eurlex_de |
|
- legal_mc4_de |
|
- wikipedia_20231101_de |
|
- wikibooks_de |
|
- wikiquote_de |
|
- wikinews_de |
|
- wikisource_de |
|
- wikivoyage_de |
|
- colossal_oscar_2015-14_de |
|
- colossal_oscar_2016-40_de |
|
- colossal_oscar_2017-43_de |
|
- colossal_oscar_2018-47_de |
|
- colossal_oscar_2019-22_de |
|
- colossal_oscar_2020-24_de |
|
- colossal_oscar_2020-45_de |
|
- colossal_oscar_2021-49_de |
|
- colossal_oscar_2022-27_de |
|
- colossal_oscar_2022-49_de |
|
- colossal_oscar_2023-14_de |
|
- colossal_oscar_2023-23_de |
|
- open_discourse_bundestag |
|
- tagesschau_2018_2023 |
|
- opensubtitles_de |
|
- parlamint_at |
|
|
|
|
|
- itwac |
|
- eurlex_it |
|
- legal_mc4_it |
|
- wikipedia_20231101_it |
|
- wikibooks_it |
|
- wikiquote_it |
|
- wikinews_it |
|
- wikisource_it |
|
- wikivoyage_it |
|
- colossal_oscar_2015-14_it |
|
- colossal_oscar_2016-40_it |
|
- colossal_oscar_2017-43_it |
|
- colossal_oscar_2018-47_it |
|
- colossal_oscar_2019-22_it |
|
- colossal_oscar_2020-24_it |
|
- colossal_oscar_2020-45_it |
|
- colossal_oscar_2021-49_it |
|
- colossal_oscar_2022-27_it |
|
- colossal_oscar_2022-49_it |
|
- colossal_oscar_2023-14_it |
|
- colossal_oscar_2023-23_it |
|
- opensubtitles_it |
|
- parlamint_it |
|
- tatoeba_translation_en_fr |
|
- tatoeba_translation_en_es |
|
- tatoeba_translation_en_it |
|
- tatoeba_translation_fr_it |
|
- tatoeba_translation_es_fr |
|
- tatoeba_translation_es_it |
|
- tatoeba_translation_de_en |
|
- tatoeba_translation_de_fr |
|
- tatoeba_translation_de_es |
|
- tatoeba_translation_de_it |
|
- opus100_translation_de_en |
|
- opus100_translation_en_es |
|
- opus100_translation_en_fr |
|
- opus100_translation_en_it |
|
- wmt19_translation_de_en |
|
- wmt19_translation_fr_de |
|
|
|
sampling_factor_by_dataset_id: |
|
redpajama_stackexchange: 0.1 |
|
pes2o: 0.1 |
|
math_amps: 0.1 |
|
openlegaldata: 0.75 |
|
dewac: 0.05 |
|
itwac: 1 |
|
cabernet: 1 |
|
spanish_legal: 0.1 |
|
eurlex_de: 0.5 |
|
eurlex_en: 0.5 |
|
eurlex_es: 1 |
|
eurlex_fr: 1 |
|
eurlex_it: 1 |
|
legal_mc4_de: 0.1 |
|
legal_mc4_es: 0.25 |
|
legal_mc4_fr: 0.25 |
|
legal_mc4_it: 1 |
|
wikipedia_20231101_de: 2 |
|
wikibooks_de: 1 |
|
wikiquote_de: 1 |
|
wikinews_de: 2 |
|
wikisource_de: 1 |
|
wikivoyage_de: 1 |
|
wikipedia_20231101_en: 1 |
|
wikibooks_en: 1 |
|
wikiquote_en: 0.25 |
|
wikinews_en: 1 |
|
wikisource_en: 1 |
|
wikivoyage_en: 1 |
|
wikipedia_20231101_es: 2 |
|
wikibooks_es: 1 |
|
wikiquote_es: 1 |
|
wikinews_es: 2 |
|
wikisource_es: 1 |
|
wikivoyage_es: 1 |
|
wikipedia_20231101_fr: 2 |
|
wikibooks_fr: 1 |
|
wikiquote_fr: 1 |
|
wikinews_fr: 2 |
|
wikisource_fr: 1 |
|
wikivoyage_fr: 1 |
|
wikipedia_20231101_it: 2 |
|
wikibooks_it: 1 |
|
wikiquote_it: 1 |
|
wikinews_it: 2 |
|
wikisource_it: 1 |
|
wikivoyage_it: 1 |
|
colossal_oscar_2015-14_de: 1 |
|
colossal_oscar_2016-40_de: 0.95 |
|
colossal_oscar_2017-43_de: 0.1 |
|
colossal_oscar_2018-47_de: 0.1 |
|
colossal_oscar_2019-22_de: 0.1 |
|
colossal_oscar_2020-24_de: 0.1 |
|
colossal_oscar_2020-45_de: 0.1 |
|
colossal_oscar_2021-49_de: 0.1 |
|
colossal_oscar_2022-27_de: 0.1 |
|
colossal_oscar_2022-49_de: 0.1 |
|
colossal_oscar_2023-14_de: 0.95 |
|
colossal_oscar_2023-23_de: 1 |
|
colossal_oscar_2015-14_en: 0.05 |
|
colossal_oscar_2016-40_en: 0.05 |
|
colossal_oscar_2017-43_en: 0.001 |
|
colossal_oscar_2018-47_en: 0.001 |
|
colossal_oscar_2019-22_en: 0.001 |
|
colossal_oscar_2020-24_en: 0.001 |
|
colossal_oscar_2020-45_en: 0.001 |
|
colossal_oscar_2021-49_en: 0.001 |
|
colossal_oscar_2022-27_en: 0.001 |
|
colossal_oscar_2022-49_en: 0.001 |
|
colossal_oscar_2023-14_en: 0.05 |
|
colossal_oscar_2023-23_en: 0.05 |
|
colossal_oscar_2015-14_es: 1 |
|
colossal_oscar_2016-40_es: 1 |
|
colossal_oscar_2017-43_es: 0.25 |
|
colossal_oscar_2018-47_es: 0.1 |
|
colossal_oscar_2019-22_es: 0.1 |
|
colossal_oscar_2020-24_es: 0.1 |
|
colossal_oscar_2020-45_es: 0.1 |
|
colossal_oscar_2021-49_es: 0.1 |
|
colossal_oscar_2022-27_es: 0.1 |
|
colossal_oscar_2022-49_es: 0.3 |
|
colossal_oscar_2023-14_es: 1 |
|
colossal_oscar_2023-23_es: 1 |
|
colossal_oscar_2015-14_fr: 1 |
|
colossal_oscar_2016-40_fr: 1 |
|
colossal_oscar_2017-43_fr: 0.25 |
|
colossal_oscar_2018-47_fr: 0.25 |
|
colossal_oscar_2019-22_fr: 0.1 |
|
colossal_oscar_2020-24_fr: 0.1 |
|
colossal_oscar_2020-45_fr: 0.1 |
|
colossal_oscar_2021-49_fr: 0.1 |
|
colossal_oscar_2022-27_fr: 0.1 |
|
colossal_oscar_2022-49_fr: 0.75 |
|
colossal_oscar_2023-14_fr: 1 |
|
colossal_oscar_2023-23_fr: 1 |
|
starcoder_emacs-lisp: 0.1 |
|
starcoder_literate-haskell: 0.1 |
|
starcoder_shell: 0.1 |
|
starcoder_ada: 0.1 |
|
starcoder_erlang: 0.1 |
|
starcoder_lua: 0.1 |
|
starcoder_smalltalk: 0.1 |
|
starcoder_agda: 0.1 |
|
starcoder_f-sharp: 0.1 |
|
starcoder_makefile: 0.1 |
|
starcoder_solidity: 0.1 |
|
starcoder_alloy: 0.1 |
|
starcoder_fortran: 0.1 |
|
starcoder_maple: 0.1 |
|
starcoder_sparql: 0.1 |
|
starcoder_antlr: 0.1 |
|
starcoder_git-commits-cleaned: 0.05 |
|
starcoder_markdown: 0.05 |
|
starcoder_sql: 0.1 |
|
starcoder_applescript: 0.1 |
|
starcoder_github-issues-filtered-structured: 0.075 |
|
starcoder_mathematica: 0.1 |
|
starcoder_stan: 0.1 |
|
starcoder_assembly: 0.1 |
|
starcoder_glsl: 0.1 |
|
starcoder_matlab: 0.1 |
|
starcoder_standard-ml: 0.1 |
|
starcoder_augeas: 0.1 |
|
starcoder_go: 0.05 |
|
starcoder_ocaml: 0.1 |
|
starcoder_stata: 0.1 |
|
starcoder_awk: 0.1 |
|
starcoder_groovy: 0.1 |
|
starcoder_pascal: 0.1 |
|
starcoder_systemverilog: 0.1 |
|
starcoder_batchfile: 0.1 |
|
starcoder_haskell: 0.1 |
|
starcoder_perl: 0.1 |
|
starcoder_tcl: 0.1 |
|
starcoder_bluespec: 0.1 |
|
starcoder_html: 0.05 |
|
starcoder_php: 0.05 |
|
starcoder_tcsh: 0.1 |
|
starcoder_c: 0.05 |
|
starcoder_idris: 0.1 |
|
starcoder_powershell: 0.1 |
|
starcoder_tex: 0.1 |
|
starcoder_c-sharp: 0.05 |
|
starcoder_isabelle: 0.1 |
|
starcoder_prolog: 0.1 |
|
starcoder_thrift: 0.1 |
|
starcoder_clojure: 0.1 |
|
starcoder_java: 0.05 |
|
starcoder_protocol-buffer: 0.1 |
|
starcoder_typescript: 0.05 |
|
starcoder_cmake: 0.1 |
|
starcoder_java-server-pages: 0.1 |
|
starcoder_python: 0.05 |
|
starcoder_verilog: 0.1 |
|
starcoder_coffeescript: 0.1 |
|
starcoder_javascript: 0.05 |
|
starcoder_r: 0.1 |
|
starcoder_vhdl: 0.1 |
|
starcoder_common-lisp: 0.1 |
|
starcoder_json: 0.1 |
|
starcoder_racket: 0.1 |
|
starcoder_visual-basic: 0.1 |
|
starcoder_cpp: 0.05 |
|
starcoder_julia: 0.1 |
|
starcoder_restructuredtext: 0.1 |
|
starcoder_xslt: 0.1 |
|
starcoder_css: 0.1 |
|
starcoder_jupyter-scripts-dedup-filtered: 0.1 |
|
starcoder_rmarkdown: 0.1 |
|
starcoder_yacc: 0.1 |
|
starcoder_cuda: 0.1 |
|
starcoder_jupyter-structured-clean-dedup: 0.1 |
|
starcoder_ruby: 0.1 |
|
starcoder_yaml: 0.1 |
|
starcoder_dart: 0.1 |
|
starcoder_kotlin: 0.1 |
|
starcoder_rust: 0.1 |
|
starcoder_zig: 0.1 |
|
starcoder_dockerfile: 0.1 |
|
starcoder_lean: 0.1 |
|
starcoder_sas: 0.1 |
|
starcoder_elixir: 0.1 |
|
starcoder_literate-agda: 0.1 |
|
starcoder_scala: 0.1 |
|
starcoder_elm: 0.1 |
|
starcoder_literate-coffeescript: 0.1 |
|
starcoder_scheme: 0.1 |
|
pile_of_law_r_legaladvice: 1 |
|
pile_of_law_atticus_contracts: 0.25 |
|
pile_of_law_un_debates: 1 |
|
open_discourse_bundestag: 0.5 |
|
tagesschau_2018_2023: 1 |
|
proof_pile2_open_web_math: 0.25 |
|
tatoeba_translation_en_fr: 1 |
|
tatoeba_translation_en_es: 1 |
|
tatoeba_translation_en_it: 1 |
|
tatoeba_translation_fr_it: 1 |
|
tatoeba_translation_es_fr: 1 |
|
tatoeba_translation_es_it: 1 |
|
tatoeba_translation_de_en: 1 |
|
tatoeba_translation_de_fr: 1 |
|
tatoeba_translation_de_es: 1 |
|
tatoeba_translation_de_it: 1 |
|
opus100_translation_de_en: 1 |
|
opus100_translation_en_es: 1 |
|
opus100_translation_en_fr: 1 |
|
opus100_translation_en_it: 1 |
|
wmt19_translation_de_en: 1 |
|
wmt19_translation_fr_de: 1 |
|
opensubtitles_es: 1 |
|
opensubtitles_fr: 1 |
|
opensubtitles_de: 1 |
|
opensubtitles_it: 1 |
|
parlamint_es: 1 |
|
parlamint_it: 1 |
|
parlamint_at: 1 |
|
parlamint_fr: 1 |
|
parlamint_gb: 1 |
|
colossal_oscar_2015-14_it: 1 |
|
colossal_oscar_2016-40_it: 1 |
|
colossal_oscar_2017-43_it: 0.75 |
|
colossal_oscar_2018-47_it: 0.75 |
|
colossal_oscar_2019-22_it: 0.75 |
|
colossal_oscar_2020-24_it: 0.75 |
|
colossal_oscar_2020-45_it: 0.75 |
|
colossal_oscar_2021-49_it: 0.75 |
|
colossal_oscar_2022-27_it: 0.75 |
|
colossal_oscar_2022-49_it: 0.75 |
|
colossal_oscar_2023-14_it: 0.9 |
|
colossal_oscar_2023-23_it: 1 |
|
|
