Spaces:
Running
Running
Commit
•
0067690
1
Parent(s):
8c4a00c
Remove Dolma 1.6
Browse files- constants.py +1 -1
constants.py
CHANGED
|
@@ -3,7 +3,6 @@ import os
|
|
| 3 |
# options
|
| 4 |
INDEX_BY_DESC = {
|
| 5 |
'Dolma-v1.7 (2.6T tokens)': 'v4_dolma-v1_7_llama',
|
| 6 |
-
'Dolma-v1.6 (3.1T tokens)': 'v4_dolma-v1_6_llama',
|
| 7 |
'RedPajama (1.4T tokens)': 'v4_rpj_llama_s4',
|
| 8 |
'Pile-train (380B tokens)': 'v4_piletrain_llama',
|
| 9 |
'C4-train (200B tokens)': 'v4_c4train_llama',
|
|
@@ -11,6 +10,7 @@ INDEX_BY_DESC = {
|
|
| 11 |
# 'Pile-val (GPT-2 tokenizer), 380M tokens': 'v4_pileval_gpt2',
|
| 12 |
# 'Dolma-v1.6-sample (OLMo tokenizer), 8.0B tokens': 'v4_dolmasample_olmo',
|
| 13 |
# 'Dolma-v1.6-sample (9.2B tokens)': 'v4_dolma-v1_6-sample_llama',
|
|
|
|
| 14 |
# 'Dolma-v1.6-wiki (4.3B tokens)': 'v4_dolma-v1_6-wiki_llama',
|
| 15 |
# 'Dolma-v1.6-books (5.8B tokens)': 'v4_dolma-v1_6-books_llama',
|
| 16 |
# 'Dolma-v1.6-pes2o (69B tokens)': 'v4_dolma-v1_6-pes2o_llama',
|
|
|
|
| 3 |
# options
|
| 4 |
INDEX_BY_DESC = {
|
| 5 |
'Dolma-v1.7 (2.6T tokens)': 'v4_dolma-v1_7_llama',
|
|
|
|
| 6 |
'RedPajama (1.4T tokens)': 'v4_rpj_llama_s4',
|
| 7 |
'Pile-train (380B tokens)': 'v4_piletrain_llama',
|
| 8 |
'C4-train (200B tokens)': 'v4_c4train_llama',
|
|
|
|
| 10 |
# 'Pile-val (GPT-2 tokenizer), 380M tokens': 'v4_pileval_gpt2',
|
| 11 |
# 'Dolma-v1.6-sample (OLMo tokenizer), 8.0B tokens': 'v4_dolmasample_olmo',
|
| 12 |
# 'Dolma-v1.6-sample (9.2B tokens)': 'v4_dolma-v1_6-sample_llama',
|
| 13 |
+
# 'Dolma-v1.6 (3.1T tokens)': 'v4_dolma-v1_6_llama',
|
| 14 |
# 'Dolma-v1.6-wiki (4.3B tokens)': 'v4_dolma-v1_6-wiki_llama',
|
| 15 |
# 'Dolma-v1.6-books (5.8B tokens)': 'v4_dolma-v1_6-books_llama',
|
| 16 |
# 'Dolma-v1.6-pes2o (69B tokens)': 'v4_dolma-v1_6-pes2o_llama',
|