fixed smaller pretrain dataset

scripts/prepare_pretrain_dataset.py

@@ -20,7 +20,7 @@ def batch_iterator(name=None):
         for d in dataset:
             for row in d:
                 for n in row:
-                    yield row['instruction'] + '\n' + row['input'] + '\n' + row['output']
+                    yield row['instruction'] + ' ' + row['input'] + ' ' + row['output']
 
         del dataset
         gc.collect()
@@ -79,7 +79,7 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
         
         for row in dataset:
-            yield row['prompt'] + '\n' + row['response']
+            yield row['prompt'] + ' ' + row['response']
         
         del dataset
         gc.collect()
@@ -120,7 +120,7 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
 
         for row in dataset:
-            yield row['query'] + '\n' + row['answer']
+            yield row['query'] + ' ' + row['answer']
 
         del dataset
         gc.collect()
@@ -130,7 +130,7 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
 
         for row in dataset:
-            yield row['instruction'] + '\n' + row['output']
+            yield row['instruction'] + ' ' + row['output']
 
         del dataset
         gc.collect()
@@ -140,7 +140,7 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
 
         for row in dataset:
-            yield row['instruction'] + '\n' + row['input'] + '\n' + row['output']
+            yield row['instruction'] + ' ' + row['input'] + ' ' + row['output']
 
         del dataset
         gc.collect()
@@ -150,7 +150,7 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
 
         for row in dataset:
-            yield row['prompt'] + '\n' + row['completion']
+            yield row['prompt'] + ' ' + row['completion']
 
         del dataset
         gc.collect()
@@ -171,7 +171,7 @@ def batch_iterator(name=None):
             dataset = load_dataset(name, split=split)
             
             for row in dataset:
-                yield row['question'] + '\n' + row['answer']
+                yield row['question'] + ' ' + row['answer']
             
             del dataset
             gc.collect()
@@ -181,7 +181,7 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
         
         for row in dataset:
-            yield row['instruction'] + '\n' + row['output']
+            yield row['instruction'] + ' ' + row['output']
         
         del dataset
         gc.collect()
@@ -191,7 +191,7 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
 
         for row in dataset:
-            yield row['question'] + '\n' + row['answer']
+            yield row['question'] + ' ' + row['answer']
 
         del dataset
         gc.collect()
@@ -201,7 +201,7 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
 
         for row in dataset:
-            yield row['question_translated_srb'] + '\n' + row['answer_translated_srb']
+            yield row['question_translated_srb'] + ' ' + row['answer_translated_srb']
 
         del dataset
         gc.collect()
@@ -211,7 +211,13 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
         
         for row in dataset:
-            yield f'{row["character"]}\n{row["unicode"]}\n{row["short description"]}\n{row["tags"]}\n{row["LLM description"]}'
+            yield (
+                row['character'] + ' ' +
+                row['unicode'] + ' ' +
+                row['short description'] + ' ' +
+                row['tags'] + ' ' +
+                row['LLM description']
+            )
         
         del dataset
         gc.collect()