pretrain dataset
Browse files
scripts/prepare_pretrain_dataset.py
CHANGED
@@ -84,7 +84,7 @@ def batch_iterator(name=None):
|
|
84 |
|
85 |
# text
|
86 |
if name in (None, 'yirenc/general_knowledge_boolean'):
|
87 |
-
for split in ['train', '
|
88 |
dataset = load_dataset(name, split=split)
|
89 |
|
90 |
for row in dataset:
|
|
|
84 |
|
85 |
# text
|
86 |
if name in (None, 'yirenc/general_knowledge_boolean'):
|
87 |
+
for split in ['train', 'validation']:
|
88 |
dataset = load_dataset(name, split=split)
|
89 |
|
90 |
for row in dataset:
|