pretrain model

scripts/prepare_contrain_dataset.py

@@ -28,4 +28,6 @@ https://huggingface.co/datasets/KingNish/reasoning-base-20k
 https://huggingface.co/datasets/Magpie-Align/Magpie-Reasoning-150K
 https://huggingface.co/datasets/ai2-adapt-dev/openmath-2-math
 https://huggingface.co/datasets/thesven/gsm8k-reasoning
+
+https://huggingface.co/datasets/codeparrot/self-instruct-starcoder
 """

scripts/prepare_pretrain_dataset.py

@@ -55,277 +55,68 @@ datasets_configs = [
         for data_dir in [
             f'multilingual-instruction-tuning-dataset /multilingual-alpaca-52k-gpt-4/{n}'
             for n in [
-                'Afrikaans',
-                'Albanian',
-                'Amharic',
-                'Arabic',
-                'Armenian',
-                'Assamese',
-                'Aymara',
-                'Azerbaijani',
-                'Bambara',
-                'Basque',
-                'Belarusian',
-                'Bengali',
-                'Bhojpuri',
-                'Bosnian',
-                'Bulgarian',
-                'Catalan',
-                'Cebuano',
-                'Chichewa',
-                'ChineseSimplified',
-                'ChineseTraditional',
-                'Corsican',
-                'Croatian',
-                'Czech',
-                'Danish',
-                'Divehi',
-                'Dogri',
-                'Dutch',
-                'Esperanto',
-                'Estonian',
-                'Ewe',
-                'Filipino',
-                'Finnish',
-                'French',
-                'Frisian',
-                'Galician',
-                'Georgian',
-                'German',
-                'Greek',
-                'Guarani',
-                'Gujarati',
-                'Haitian_Creole',
-                'Hausa',
-                'Hawaiian',
-                'Hebrew',
-                'Hindi',
-                'Hmong',
-                'Hungarian',
-                'Icelandic',
-                'Igbo',
-                'Ilocano',
-                'Indonesian',
-                'Irish',
-                'Italian',
-                'Japanese',
-                'Javanese',
-                'Kannada',
-                'Kazakh',
-                'Khmer',
-                'Kinyarwanda',
-                'Konkani',
-                'Korean',
-                'Krio',
-                'Kurdish_Kurmanji',
-                'Kurdish_Sorani',
-                'Kyrgyz',
-                'Lao',
-                'Latin',
-                'Latvian',
-                'Lingala',
-                'Lithuanian',
-                'Luganda',
-                'Luxembourgish',
-                'Macedonian',
-                'Maithili',
-                'Malagasy',
-                'Malay',
-                'Malayalam',
-                'Maltese',
-                'Maori',
-                'Marathi',
-                'Meiteilon_Manipuri',
-                'Mizo',
-                'Mongolian',
-                'Myanmar_Burmese',
-                'Nepali',
-                'Norwegian',
-                'Odia_Oriya',
-                'Oromo',
-                'Pashto',
-                'Persian',
-                'Polish',
-                'Portuguese',
-                'Punjabi',
-                'Quechua',
-                'Romanian',
-                'Russian',
-                'Samoan',
-                'Sanskrit',
-                'ScottishGaelic',
-                'Sepedi',
-                'Serbian',
-                'Sesotho',
-                'Shona',
-                'Sindhi',
-                'Sinhala',
-                'Slovak',
-                'Slovenian',
-                'Somali',
-                'Spanish',
-                'Sundanese',
-                'Swahili',
-                'Swedish',
-                'Tajik',
-                'Tamil',
-                'Tatar',
-                'Telugu',
-                'Thai',
-                'Tigrinya',
-                'Tsonga',
-                'Turkish',
-                'Turkmen',
-                'Twi',
-                'Ukrainian',
-                'Urdu',
-                'Uyghur',
-                'Uzbek',
-                'Vietnamese',
-                'Welsh',
-                'Xhosa',
-                'Yiddish',
-                'Yoruba',
-                'Zulu',
+                'Afrikaans', 'Albanian', 'Amharic', 'Arabic', 'Armenian', 'Assamese',
+                'Aymara', 'Azerbaijani', 'Bambara', 'Basque', 'Belarusian', 'Bengali',
+                'Bhojpuri', 'Bosnian', 'Bulgarian', 'Catalan', 'Cebuano', 'Chichewa',
+                'ChineseSimplified', 'ChineseTraditional', 'Corsican', 'Croatian',
+                'Czech', 'Danish', 'Divehi', 'Dogri', 'Dutch', 'Esperanto', 'Estonian',
+                'Ewe', 'Filipino', 'Finnish', 'French', 'Frisian', 'Galician',
+                'Georgian', 'German', 'Greek', 'Guarani', 'Gujarati', 'Haitian_Creole',
+                'Hausa', 'Hawaiian', 'Hebrew', 'Hindi', 'Hmong', 'Hungarian',
+                'Icelandic', 'Igbo', 'Ilocano', 'Indonesian', 'Irish', 'Italian',
+                'Japanese', 'Javanese', 'Kannada', 'Kazakh', 'Khmer', 'Kinyarwanda',
+                'Konkani', 'Korean', 'Krio', 'Kurdish_Kurmanji', 'Kurdish_Sorani',
+                'Kyrgyz', 'Lao', 'Latin', 'Latvian', 'Lingala', 'Lithuanian',
+                'Luganda', 'Luxembourgish', 'Macedonian', 'Maithili', 'Malagasy',
+                'Malay', 'Malayalam', 'Maltese', 'Maori', 'Marathi', 'Meiteilon_Manipuri',
+                'Mizo', 'Mongolian', 'Myanmar_Burmese', 'Nepali', 'Norwegian',
+                'Odia_Oriya', 'Oromo', 'Pashto', 'Persian', 'Polish', 'Portuguese',
+                'Punjabi', 'Quechua', 'Romanian', 'Russian', 'Samoan', 'Sanskrit',
+                'ScottishGaelic', 'Sepedi', 'Serbian', 'Sesotho', 'Shona', 'Sindhi',
+                'Sinhala', 'Slovak', 'Slovenian', 'Somali', 'Spanish', 'Sundanese',
+                'Swahili', 'Swedish', 'Tajik', 'Tamil', 'Tatar', 'Telugu', 'Thai',
+                'Tigrinya', 'Tsonga', 'Turkish', 'Turkmen', 'Twi', 'Ukrainian',
+                'Urdu', 'Uyghur', 'Uzbek', 'Vietnamese', 'Welsh', 'Xhosa',
+                'Yiddish', 'Yoruba', 'Zulu',
             ]
         ]
     ],
     *[
         {'path': 'saillab/taco-datasets', 'data_dir': 'multilingual-instruction-tuning-dataset /multilinugal-dolly-15k/', 'data_files': n, 'split': f'train', 'format': '{instruction} {input} {output}'}
         for n in [
-            'Afrikaans.json',
-            'Albanian.json',
-            'Amharic.json',
-            'Arabic.json',
-            'Armenian.json',
-            'Assamese.json',
-            'Aymara.json',
-            'Azerbaijani.json',
-            'Bambara.json',
-            'Basque.json',
-            'Belarusian.json',
-            'Bengali.json',
-            'Bhojpuri.json',
-            'Bosnian.json',
-            'Bulgarian.json',
-            'Catalan.json',
-            'Cebuano.json',
-            'Chichewa.json',
-            'ChineseSimplified.json',
-            'ChineseTraditional.json',
-            'Corsican.json',
-            'Croatian.json',
-            'Czech.json',
-            'Danish.json',
-            'Dhivehi.json',
-            'Dogri.json',
-            'Dutch.json',
-            'English.json',
-            'Esperanto.json',
-            'Estonian.json',
-            'Ewe.json',
-            'Filipino.json',
-            'Finnish.json',
-            'French.json',
-            'Frisian.json',
-            'Galician.json',
-            'Georgian.json',
-            'German.json',
-            'Greek.json',
-            'Guarani.json',
-            'Gujarati.json',
-            'Haitian_Creole.json',
-            'Hausa.json',
-            'Hawaiian.json',
-            'Hebrew.json',
-            'Hindi.json',
-            'Hmong.json',
-            'Hungarian.json',
-            'Icelandic.json',
-            'Igbo.json',
-            'Ilocano.json',
-            'Indonesian.json',
-            'Irish.json',
-            'Italian.json',
-            'Japanese.json',
-            'Javanese.json',
-            'Kannada.json',
-            'Kazakh.json',
-            'Khmer.json',
-            'Kinyarwanda.json',
-            'Konkani.json',
-            'Korean.json',
-            'Krio.json',
-            'Kurdish_Kurmanji.json',
-            'Kurdish_Sorani.json',
-            'Kyrgyz.json',
-            'Lao.json',
-            'Latin.json',
-            'Latvian.json',
-            'Lingala.json',
-            'Lithuanian.json',
-            'Luganda.json',
-            'Luxembourgish.json',
-            'Macedonian.json',
-            'Maithili.json',
-            'Malagasy.json',
-            'Malayalam.json',
-            'Malay.json',
-            'Maltese.json',
-            'Maori.json',
-            'Marathi.json',
-            'Meiteilon_Manipuri.json',
-            'Mizo.json',
-            'Mongolian.json',
-            'Myanmar_Burmese.json',
-            'Nepali.json',
-            'Norwegian.json',
-            'Odia_Oriya.json',
-            'Oromo.json',
-            'Pashto.json',
-            'Persian.json',
-            'Polish.json',
-            'Portuguese.json',
-            'Punjabi.json',
-            'Quechua.json',
-            'Romanian.json',
-            'Russian.json',
-            'Samoan.json',
-            'Sanskrit.json',
-            'ScottishGaelic.json',
-            'Sepedi.json',
-            'Serbian.json',
-            'Sesotho.json',
-            'Shona.json',
-            'Sindhi.json',
-            'Sinhala.json',
-            'Slovak.json',
-            'Slovenian.json',
-            'Somali.json',
-            'Spanish.json',
-            'Sundanese.json',
-            'Swahili.json',
-            'Swedish.json',
-            'Tajik.json',
-            'Tamil.json',
-            'Tatar.json',
-            'Telugu.json',
-            'Thai.json',
-            'Tigrinya.json',
-            'Tsonga.json',
-            'Turkish.json',
-            'Turkmen.json',
-            'Twi.json',
-            'Ukrainian.json',
-            'Urdu.json',
-            'Uyghur.json',
-            'Uzbek.json',
-            'Vietnamese.json',
-            'Welsh.json',
-            'Xhosa.json',
-            'Yiddish.json',
-            'Yoruba.json',
-            'Zulu.json',
+            'Afrikaans.json', 'Albanian.json', 'Amharic.json', 'Arabic.json', 'Armenian.json',
+            'Assamese.json', 'Aymara.json', 'Azerbaijani.json', 'Bambara.json', 'Basque.json',
+            'Belarusian.json', 'Bengali.json', 'Bhojpuri.json', 'Bosnian.json', 'Bulgarian.json',
+            'Catalan.json', 'Cebuano.json', 'Chichewa.json', 'ChineseSimplified.json',
+            'ChineseTraditional.json', 'Corsican.json', 'Croatian.json', 'Czech.json',
+            'Danish.json', 'Dhivehi.json', 'Dogri.json', 'Dutch.json', 'English.json',
+            'Esperanto.json', 'Estonian.json', 'Ewe.json', 'Filipino.json',
+            'Finnish.json', 'French.json', 'Frisian.json', 'Galician.json',
+            'Georgian.json', 'German.json', 'Greek.json', 'Guarani.json',
+            'Gujarati.json', 'Haitian_Creole.json', 'Hausa.json', 'Hawaiian.json',
+            'Hebrew.json', 'Hindi.json', 'Hmong.json', 'Hungarian.json',
+            'Icelandic.json', 'Igbo.json', 'Ilocano.json', 'Indonesian.json',
+            'Irish.json', 'Italian.json', 'Japanese.json', 'Javanese.json',
+            'Kannada.json', 'Kazakh.json', 'Khmer.json', 'Kinyarwanda.json',
+            'Konkani.json', 'Korean.json', 'Krio.json', 'Kurdish_Kurmanji.json',
+            'Kurdish_Sorani.json', 'Kyrgyz.json', 'Lao.json', 'Latin.json',
+            'Latvian.json', 'Lingala.json', 'Lithuanian.json', 'Luganda.json',
+            'Luxembourgish.json', 'Macedonian.json', 'Maithili.json',
+            'Malagasy.json', 'Malayalam.json', 'Malay.json', 'Maltese.json',
+            'Maori.json', 'Marathi.json', 'Meiteilon_Manipuri.json',
+            'Mizo.json', 'Mongolian.json', 'Myanmar_Burmese.json',
+            'Nepali.json', 'Norwegian.json', 'Odia_Oriya.json', 'Oromo.json',
+            'Pashto.json', 'Persian.json', 'Polish.json', 'Portuguese.json',
+            'Punjabi.json', 'Quechua.json', 'Romanian.json', 'Russian.json',
+            'Samoan.json', 'Sanskrit.json', 'ScottishGaelic.json', 'Sepedi.json',
+            'Serbian.json', 'Sesotho.json', 'Shona.json', 'Sindhi.json',
+            'Sinhala.json', 'Slovak.json', 'Slovenian.json', 'Somali.json',
+            'Spanish.json', 'Sundanese.json', 'Swahili.json', 'Swedish.json',
+            'Tajik.json', 'Tamil.json', 'Tatar.json', 'Telugu.json', 'Thai.json',
+            'Tigrinya.json', 'Tsonga.json', 'Turkish.json', 'Turkmen.json',
+            'Twi.json', 'Ukrainian.json', 'Urdu.json', 'Uyghur.json', 'Uzbek.json',
+            'Vietnamese.json', 'Welsh.json', 'Xhosa.json', 'Yiddish.json',
+            'Yoruba.json', 'Zulu.json',
         ]
     ],
     *[

scripts/pretrain-model.yaml

@@ -67,7 +67,7 @@ train:
 
   # Number of samples per data-parallel rank (type: int, default: 4)
   # micro_batch_size: 16
-  micro_batch_size: 4
+  micro_batch_size: 12
 
   # Number of iterations with learning rate warmup active (type: int, default: 2000)
   lr_warmup_steps: 2000