Update tokenizer_config.json

This removes the extra tokens whose indices exceed the vocab size. Only the pad token is actually used, so we do the standard thing of using the EOS token as the PAD token.

Further, we set the `model_max_length`, which was previously not set, as well as changing the `padding_side` to 'left' instead of 'right', as the model is auto-regressive.

tokenizer_config.json

@@ -26,57 +26,14 @@
       "single_word": false,
       "special": true
     },
-    "32000": {
-      "content": "<CLS>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32001": {
-      "content": "<SEP>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32002": {
-      "content": "<EOD>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32003": {
-      "content": "<MASK>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32004": {
-      "content": "<PAD>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
   },
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
-  "cls_token": "<CLS>",
   "eos_token": "</s>",
   "legacy": false,
-  "mask_token": "<MASK>",
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<PAD>",
-  "padding_side": "right",
-  "sep_token": "<SEP>",
+  "model_max_length": 4096,
+  "pad_token": "</s>",
+  "padding_side": "left",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",