pretrain dataset
Browse files
scripts/prepare_pretrain_dataset.py
CHANGED
@@ -76,7 +76,7 @@ def batch_iterator(name=None):
|
|
76 |
for row in dataset:
|
77 |
if not row['Question'] or not row['Answer']:
|
78 |
continue
|
79 |
-
|
80 |
yield row['Question'] + ' ' + row['Answer']
|
81 |
|
82 |
del dataset
|
@@ -268,9 +268,9 @@ def batch_iterator(name=None):
|
|
268 |
|
269 |
for row in dataset:
|
270 |
yield (
|
271 |
-
row['instruction'] +
|
272 |
' = ' +
|
273 |
-
row['output']
|
274 |
)
|
275 |
|
276 |
del dataset
|
|
|
76 |
for row in dataset:
|
77 |
if not row['Question'] or not row['Answer']:
|
78 |
continue
|
79 |
+
|
80 |
yield row['Question'] + ' ' + row['Answer']
|
81 |
|
82 |
del dataset
|
|
|
268 |
|
269 |
for row in dataset:
|
270 |
yield (
|
271 |
+
str(row['instruction']) +
|
272 |
' = ' +
|
273 |
+
str(row['output'])
|
274 |
)
|
275 |
|
276 |
del dataset
|