new tokenizer 38400

merges.txt

@@ -36637,53 +36637,3 @@ aki w
 ĠÛģÙĪ تا
 ari hi
 pa i
-zi y
-Ġvolt ages
-d achadh
-Ġt xim
-Ġб аÑĤ
-ĠáĭŃáĪħ áĪĿ
-ù n
-ê¯ģ ê¯ħê¯ģ
-模 åŀĭ
-omb iso
-Ġà¤ķ à¤Ł
-Ħì Ľ
-er at
-Ġelect rod
-Ġin sect
-Ġcomp ares
-ĠÑĩ ел
-Ġeng els
-Ġgi hatag
-ĠG ẹẹsi
-×ķ׳ ×Ļ
-ul sion
-ش ر
-аÑĨ а
-Ġh ire
-us oro
-ĠBo oks
-رÙĪ Ùģ
-en ç
-Ġter ra
-Ġa za
-Ġd ance
-Ġin zawm
-Ġverb ess
-Ġuk w
-ĠB io
-Ġng ak
-r ise
-ĠìĹ Ĩ
-Ġding we
-áŀ ¯
-Ġf inger
-ĠP ul
-Ġv ann
-Ġsap ertos
-Pro bability
-Ġlic ense
-Ġclos ing
-Ġs ow
-Ġs amp

scripts/prepare_pretrain_dataset.py

@@ -20,7 +20,13 @@ def batch_iterator(name=None):
         for d in dataset:
             for row in d:
                 for n in row:
-                    yield row['instruction'] + ' ' + row['input'] + ' ' + row['output']
+                    yield (
+                        row['instruction'] +
+                        ' ' +
+                        row['input'] +
+                        ' ' +
+                        row['output']
+                    )
 
         del dataset
         gc.collect()
@@ -57,19 +63,40 @@ def batch_iterator(name=None):
     if name in (None, 'ontocord/fineweb-permissive-multilingual-2m'):
         dataset = load_dataset(name, split='train')
         
-        for row in dataset:
-            yield row['text']
+        for row in dataset['text']:
+            yield row
         
         del dataset
         gc.collect()
 
     # text
-    if name in (None, 'nampdn-ai/tiny-textbooks'):
+    if name in (None, 'MuskumPillerum/General-Knowledge'):
+        dataset = load_dataset(name, split='train')
+
+        for row in dataset:
+            yield row['Question'] + ' ' + row['Answer']
+
+        del dataset
+        gc.collect()
+
+    # text
+    if name in (None, 'yirenc/general_knowledge_boolean'):
         for split in ['train', 'test']:
             dataset = load_dataset(name, split=split)
 
             for row in dataset:
-                yield row['textbook']
+                yield row['question'] + '? ' + row['answer'] + '. ' + row['passage']
+
+            del dataset
+            gc.collect()
+
+    # text
+    if name in (None, 'nampdn-ai/tiny-textbooks'):
+        for split in ['train', 'test']:
+            dataset = load_dataset(name, split=split)
+
+            for row in dataset['textbook']:
+                yield row
 
             del dataset
             gc.collect()
@@ -79,7 +106,11 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
         
         for row in dataset:
-            yield row['prompt'] + ' ' + row['response']
+            yield (
+                row['prompt'] +
+                ' ' +
+                row['response']
+            )
         
         del dataset
         gc.collect()
@@ -109,8 +140,8 @@ def batch_iterator(name=None):
         )
 
         for d in dataset:
-            for row in d:
-                yield row['content']
+            for row in d['content']:
+                yield row
 
         del dataset
         gc.collect()
@@ -120,7 +151,11 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
 
         for row in dataset:
-            yield row['query'] + ' ' + row['answer']
+            yield (
+                row['query'] +
+                ' ' +
+                row['answer']
+            )
 
         del dataset
         gc.collect()
@@ -130,7 +165,11 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
 
         for row in dataset:
-            yield row['instruction'] + ' ' + row['output']
+            yield (
+                row['instruction'] +
+                ' ' +
+                row['output']
+            )
 
         del dataset
         gc.collect()
@@ -140,27 +179,38 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
 
         for row in dataset:
-            yield row['instruction'] + ' ' + row['input'] + ' ' + row['output']
+            yield (
+                row['instruction'] +
+                ' ' +
+                row['input'] +
+                ' ' +
+                row['output']
+            )
 
         del dataset
         gc.collect()
 
     # code
     if name in (None, 'HuggingFaceH4/CodeAlpaca_20K'):
-        dataset = load_dataset(name, split='train')
+        for split in ['train', 'test']:
+            dataset = load_dataset(name, split=split)
 
-        for row in dataset:
-            yield row['prompt'] + ' ' + row['completion']
+            for row in dataset:
+                yield (
+                    row['prompt'] +
+                    ' ' +
+                    row['completion']
+                )
 
-        del dataset
-        gc.collect()
+            del dataset
+            gc.collect()
 
     # math
     if name in (None, 'gair-prox/open-web-math-pro'):
         dataset = load_dataset(name, split='train')
         
-        for row in dataset:
-            yield row['text']
+        for row in dataset['text']:
+            yield row
         
         del dataset
         gc.collect()
@@ -171,7 +221,11 @@ def batch_iterator(name=None):
             dataset = load_dataset(name, split=split)
             
             for row in dataset:
-                yield row['question'] + ' ' + row['answer']
+                yield (
+                    row['question'] +
+                    ' ' +
+                    row['answer']
+                )
             
             del dataset
             gc.collect()
@@ -181,7 +235,11 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
         
         for row in dataset:
-            yield row['instruction'] + ' ' + row['output']
+            yield (
+                row['instruction'] +
+                ' ' +
+                row['output']
+            )
         
         del dataset
         gc.collect()
@@ -191,17 +249,42 @@ def batch_iterator(name=None):
         dataset = load_dataset(name, split='train')
 
         for row in dataset:
-            yield row['question'] + ' ' + row['answer']
+            yield (
+                row['question'] +
+                ' ' +
+                row['answer']
+            )
 
         del dataset
         gc.collect()
 
-    # math serbian
-    if name in (None, 'datatab/orca_math_world_problem_200k_serbian'):
+    # math
+    if name in (None, 'fblgit/simple-math'):
+        for split in ['train', 'test']:
+            dataset = load_dataset(name, split=split)
+
+            for row in dataset:
+                yield (
+                    row['instruction'] +
+                    ' = ' +
+                    row['output']
+                )
+
+            del dataset
+            gc.collect()
+
+    # reasoning
+    if name in (None, 'SkunkworksAI/reasoning-0.01'):
         dataset = load_dataset(name, split='train')
 
         for row in dataset:
-            yield row['question_translated_srb'] + ' ' + row['answer_translated_srb']
+            yield (
+                row['instruction'] +
+                ' ' +
+                row['reasoning'] +
+                ' ' +
+                row['output']
+            )
 
         del dataset
         gc.collect()
@@ -212,10 +295,14 @@ def batch_iterator(name=None):
         
         for row in dataset:
             yield (
-                row['character'] + ' ' +
-                row['unicode'] + ' ' +
-                row['short description'] + ' ' +
-                row['tags'] + ' ' +
+                row['character'] +
+                ' ' +
+                row['unicode'] +
+                ' ' +
+                row['short description'] +
+                ' ' +
+                row['tags'] +
+                ' ' +
                 row['LLM description']
             )
         
@@ -233,17 +320,21 @@ datasets_names = [
     'saillab/taco-datasets',
     # 'xu-song/cc100-samples',
     # 'ontocord/fineweb-permissive-multilingual-2m',
+    'MuskumPillerum/General-Knowledge',
+    'yirenc/general_knowledge_boolean',
     'nampdn-ai/tiny-textbooks',
-    'nampdn-ai/tiny-codes',
+    # 'nampdn-ai/tiny-codes',
     'bigcode/the-stack-smol-xs',
     'm-a-p/CodeFeedback-Filtered-Instruction',
-    'jtatman/python-code-dataset-500k',
+    # 'jtatman/python-code-dataset-500k',
     'iamtarun/python_code_instructions_18k_alpaca',
     'HuggingFaceH4/CodeAlpaca_20K',
     # 'gair-prox/open-web-math-pro',
     'rvv-karma/Math-QA',
-    'ajibawa-2023/Maths-College',
+    # 'ajibawa-2023/Maths-College',
     'microsoft/orca-math-word-problems-200k',
+    'fblgit/simple-math',
+    # 'SkunkworksAI/reasoning-0.01',
     'badrex/llm-emoji-dataset',
 ]
 
@@ -252,6 +343,6 @@ outputs = optimize(
     inputs=datasets_names,
     output_dir='../pretrain-data/',
     # Number of tokens to store by chunks. This is roughly 64MB of tokens per chunk.
-    chunk_size=(4097 * 4006),
+    chunk_size=(2049 * 8012),
     num_workers=16,
 )

scripts/pretrain-model.yaml

@@ -61,7 +61,7 @@ train:
   global_batch_size: 512
 
   # Number of samples per data-parallel rank (type: int, default: 4)
-  micro_batch_size: 8
+  micro_batch_size: 16
 
   # Number of iterations with learning rate warmup active (type: int, default: 2000)
   lr_warmup_steps: 2000
@@ -71,7 +71,7 @@ train:
 
   # Total number of tokens to train on (type: Optional[int], default: 3000000000000)
   # max_tokens: 3000000000000
-  max_tokens: ??? # ? * 8193 * 3
+  max_tokens: ??? # ? * 2049 * 3
 
   # Limits the number of optimizer steps to run. (type: Optional[int], default: null)
   max_steps:

scripts/train_tokenizer.py

@@ -158,8 +158,11 @@ special_tokens = [
     'system',
     'user',
     'assistant',
+    'resource',
     'tool',
+    'agent',
 
+    # tool/function calling
     '<tools>',
     '</tools>',
     '<tool_call>',
@@ -210,12 +213,70 @@ special_tokens = [
     '</function-type>',
     '<function-value>',
     '</function-value>',
+
+    # qa
+    '<qa>',
+    '</qa>',
+    '<question>',
+    '</question>',
+    '<answer>',
+    '</answer>',
+
+    # cot, tot
+    '<cot>',
+    '</cot>',
+    '<tot>',
+    '</tot>',
+    '<input>',
+    '</input>',
+    '<output>',
+    '</output>',
+    '<thoughts>',
+    '</thoughts>',
+    '<thought>',
+    '</thought>',
+    '<plans>',
+    '</plans>',
+    '<plan>',
+    '</plan>',
+    '<votes>',
+    '</votes>',
+    '<vote>',
+    '</vote>',
+    '<passages>',
+    '</passages>',
+    '<passage>',
+    '</passage>',
+    
+    # react
+    '<react>',
+    '</react>',
+    '<reasoning>',
+    '</reasoning>',
+    '<acting>',
+    '</acting>',
+    '<action>',
+    '</action>',
+    '<observation>',
+    '</observation>',
+    '<claim>',
+    '</claim>',
+
+    # reflection
+    '<thinking>',
+    '</thinking>',
+    '<step>',
+    '</step>',
+    '<reflection>',
+    '</reflection>',
+    '<output>',
+    '</output>',
 ]
 
 for i in range(2, 25):
     special_tokens.append(' ' * i)
 
-for i in range(64 - len(special_tokens)):
+for i in range(128 - len(special_tokens)):
     special_tokens.append(f'<|reserved_{i}|>')
 
 # emoji

tokenizer_config.json

@@ -65,7 +65,7 @@
       "special": true
     },
     "8": {
-      "content": "tool",
+      "content": "resource",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -73,7 +73,7 @@
       "special": true
     },
     "9": {
-      "content": "<tools>",
+      "content": "tool",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -81,7 +81,7 @@
       "special": true
     },
     "10": {
-      "content": "</tools>",
+      "content": "agent",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -89,7 +89,7 @@
       "special": true
     },
     "11": {
-      "content": "<tool_call>",
+      "content": "<tools>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -97,7 +97,7 @@
       "special": true
     },
     "12": {
-      "content": "</tool_call>",
+      "content": "</tools>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -105,7 +105,7 @@
       "special": true
     },
     "13": {
-      "content": "<tool_response>",
+      "content": "<tool_call>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -113,7 +113,7 @@
       "special": true
     },
     "14": {
-      "content": "</tool_response>",
+      "content": "</tool_call>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -121,7 +121,7 @@
       "special": true
     },
     "15": {
-      "content": "\"arguments\"",
+      "content": "<tool_response>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -129,7 +129,7 @@
       "special": true
     },
     "16": {
-      "content": "\"name\"",
+      "content": "</tool_response>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -137,7 +137,7 @@
       "special": true
     },
     "17": {
-      "content": "<arguments>",
+      "content": "\"arguments\"",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -145,7 +145,7 @@
       "special": true
     },
     "18": {
-      "content": "</arguments>",
+      "content": "\"name\"",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -153,7 +153,7 @@
       "special": true
     },
     "19": {
-      "content": "<argument>",
+      "content": "<arguments>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -161,7 +161,7 @@
       "special": true
     },
     "20": {
-      "content": "</argument>",
+      "content": "</arguments>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -169,7 +169,7 @@
       "special": true
     },
     "21": {
-      "content": "<argument-name>",
+      "content": "<argument>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -177,7 +177,7 @@
       "special": true
     },
     "22": {
-      "content": "</argument-name>",
+      "content": "</argument>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -185,7 +185,7 @@
       "special": true
     },
     "23": {
-      "content": "<argument-type>",
+      "content": "<argument-name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -193,7 +193,7 @@
       "special": true
     },
     "24": {
-      "content": "</argument-type>",
+      "content": "</argument-name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -201,7 +201,7 @@
       "special": true
     },
     "25": {
-      "content": "<argument-value>",
+      "content": "<argument-type>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -209,7 +209,7 @@
       "special": true
     },
     "26": {
-      "content": "</argument-value>",
+      "content": "</argument-type>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -217,7 +217,7 @@
       "special": true
     },
     "27": {
-      "content": "<parameter>",
+      "content": "<argument-value>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -225,7 +225,7 @@
       "special": true
     },
     "28": {
-      "content": "</parameter>",
+      "content": "</argument-value>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -233,7 +233,7 @@
       "special": true
     },
     "29": {
-      "content": "<parameter-name>",
+      "content": "<parameter>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -241,7 +241,7 @@
       "special": true
     },
     "30": {
-      "content": "</parameter-name>",
+      "content": "</parameter>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -249,7 +249,7 @@
       "special": true
     },
     "31": {
-      "content": "<parameter-type>",
+      "content": "<parameter-name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -257,7 +257,7 @@
       "special": true
     },
     "32": {
-      "content": "</parameter-type>",
+      "content": "</parameter-name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -265,7 +265,7 @@
       "special": true
     },
     "33": {
-      "content": "<parameter-value>",
+      "content": "<parameter-type>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -273,7 +273,7 @@
       "special": true
     },
     "34": {
-      "content": "</parameter-value>",
+      "content": "</parameter-type>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -281,7 +281,7 @@
       "special": true
     },
     "35": {
-      "content": "<field>",
+      "content": "<parameter-value>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -289,7 +289,7 @@
       "special": true
     },
     "36": {
-      "content": "</field>",
+      "content": "</parameter-value>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -297,7 +297,7 @@
       "special": true
     },
     "37": {
-      "content": "<field-name>",
+      "content": "<field>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -305,7 +305,7 @@
       "special": true
     },
     "38": {
-      "content": "</field-name>",
+      "content": "</field>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -313,7 +313,7 @@
       "special": true
     },
     "39": {
-      "content": "<field-type>",
+      "content": "<field-name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -321,7 +321,7 @@
       "special": true
     },
     "40": {
-      "content": "</field-type>",
+      "content": "</field-name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -329,7 +329,7 @@
       "special": true
     },
     "41": {
-      "content": "<field-value>",
+      "content": "<field-type>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -337,7 +337,7 @@
       "special": true
     },
     "42": {
-      "content": "</field-value>",
+      "content": "</field-type>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -345,7 +345,7 @@
       "special": true
     },
     "43": {
-      "content": "<name>",
+      "content": "<field-value>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -353,7 +353,7 @@
       "special": true
     },
     "44": {
-      "content": "</name>",
+      "content": "</field-value>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -361,7 +361,7 @@
       "special": true
     },
     "45": {
-      "content": "<type>",
+      "content": "<name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -369,7 +369,7 @@
       "special": true
     },
     "46": {
-      "content": "</type>",
+      "content": "</name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -377,7 +377,7 @@
       "special": true
     },
     "47": {
-      "content": "<value>",
+      "content": "<type>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -385,7 +385,7 @@
       "special": true
     },
     "48": {
-      "content": "</value>",
+      "content": "</type>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -393,7 +393,7 @@
       "special": true
     },
     "49": {
-      "content": "<function>",
+      "content": "<value>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -401,7 +401,7 @@
       "special": true
     },
     "50": {
-      "content": "</function>",
+      "content": "</value>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -409,7 +409,7 @@
       "special": true
     },
     "51": {
-      "content": "<function-name>",
+      "content": "<function>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -417,7 +417,7 @@
       "special": true
     },
     "52": {
-      "content": "</function-name>",
+      "content": "</function>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -425,7 +425,7 @@
       "special": true
     },
     "53": {
-      "content": "<function-type>",
+      "content": "<function-name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -433,7 +433,7 @@
       "special": true
     },
     "54": {
-      "content": "</function-type>",
+      "content": "</function-name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -441,7 +441,7 @@
       "special": true
     },
     "55": {
-      "content": "<function-value>",
+      "content": "<function-type>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -449,7 +449,7 @@
       "special": true
     },
     "56": {
-      "content": "</function-value>",
+      "content": "</function-type>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -457,7 +457,7 @@
       "special": true
     },
     "57": {
-      "content": "  ",
+      "content": "<function-value>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -465,7 +465,7 @@
       "special": true
     },
     "58": {
-      "content": "   ",
+      "content": "</function-value>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -473,7 +473,7 @@
       "special": true
     },
     "59": {
-      "content": "    ",
+      "content": "<qa>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -481,7 +481,7 @@
       "special": true
     },
     "60": {
-      "content": "     ",
+      "content": "</qa>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -489,7 +489,7 @@
       "special": true
     },
     "61": {
-      "content": "      ",
+      "content": "<question>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -497,7 +497,7 @@
       "special": true
     },
     "62": {
-      "content": "       ",
+      "content": "</question>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -505,7 +505,7 @@
       "special": true
     },
     "63": {
-      "content": "        ",
+      "content": "<answer>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -513,7 +513,7 @@
       "special": true
     },
     "64": {
-      "content": "         ",
+      "content": "</answer>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -521,7 +521,7 @@
       "special": true
     },
     "65": {
-      "content": "          ",
+      "content": "<cot>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -529,7 +529,7 @@
       "special": true
     },
     "66": {
-      "content": "           ",
+      "content": "</cot>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -537,7 +537,7 @@
       "special": true
     },
     "67": {
-      "content": "            ",
+      "content": "<tot>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -545,7 +545,7 @@
       "special": true
     },
     "68": {
-      "content": "             ",
+      "content": "</tot>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -553,7 +553,7 @@
       "special": true
     },
     "69": {
-      "content": "              ",
+      "content": "<input>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -561,7 +561,7 @@
       "special": true
     },
     "70": {
-      "content": "               ",
+      "content": "</input>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -569,7 +569,7 @@
       "special": true
     },
     "71": {
-      "content": "                ",
+      "content": "<output>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -577,7 +577,7 @@
       "special": true
     },
     "72": {
-      "content": "                 ",
+      "content": "</output>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -585,7 +585,7 @@
       "special": true
     },
     "73": {
-      "content": "                  ",
+      "content": "<thoughts>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -593,7 +593,7 @@
       "special": true
     },
     "74": {
-      "content": "                   ",
+      "content": "</thoughts>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -601,7 +601,7 @@
       "special": true
     },
     "75": {
-      "content": "                    ",
+      "content": "<thought>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -609,7 +609,7 @@
       "special": true
     },
     "76": {
-      "content": "                     ",
+      "content": "</thought>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -617,7 +617,7 @@
       "special": true
     },
     "77": {
-      "content": "                      ",
+      "content": "<plans>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -625,7 +625,7 @@
       "special": true
     },
     "78": {
-      "content": "                       ",
+      "content": "</plans>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -633,6 +633,406 @@
       "special": true
     },
     "79": {
+      "content": "<plan>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "80": {
+      "content": "</plan>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "81": {
+      "content": "<votes>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "82": {
+      "content": "</votes>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "83": {
+      "content": "<vote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "84": {
+      "content": "</vote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "85": {
+      "content": "<passages>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "86": {
+      "content": "</passages>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "87": {
+      "content": "<passage>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "88": {
+      "content": "</passage>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "89": {
+      "content": "<react>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "90": {
+      "content": "</react>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "91": {
+      "content": "<reasoning>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92": {
+      "content": "</reasoning>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "93": {
+      "content": "<acting>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "94": {
+      "content": "</acting>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "95": {
+      "content": "<action>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "96": {
+      "content": "</action>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "97": {
+      "content": "<observation>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "98": {
+      "content": "</observation>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "99": {
+      "content": "<claim>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "</claim>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "<thinking>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "</thinking>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "<step>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "104": {
+      "content": "</step>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "105": {
+      "content": "<reflection>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "106": {
+      "content": "</reflection>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "109": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "111": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "112": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "113": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "114": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "115": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "116": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "117": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "118": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "119": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "120": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "121": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "122": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "123": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "124": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "125": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "126": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "127": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "129": {
       "content": "                        ",
       "lstrip": false,
       "normalized": false,