teknium
/

Mistral-Trismegistus-7B

Text Generation

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

TheBloke commited on Oct 7, 2023

Commit

e0316eb

•

1 Parent(s): 8b0bf33

Mistral tokenizer_config.json fix

Files changed (1) hide show

tokenizer_config.json +6 -8

tokenizer_config.json CHANGED Viewed

@@ -4,25 +4,25 @@
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
-      "lstrip": true,
       "normalized": false,
-      "rstrip": true,
       "single_word": false,
       "special": true
     },
     "1": {
       "content": "<s>",
-      "lstrip": true,
       "normalized": false,
-      "rstrip": true,
       "single_word": false,
       "special": true
     },
     "2": {
       "content": "</s>",
-      "lstrip": true,
       "normalized": false,
-      "rstrip": true,
       "single_word": false,
       "special": true
     }
@@ -37,9 +37,7 @@
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
-  "tokenizer_file": "/home/teknium/.cache/huggingface/hub/models--mistralai--Mistral-7B-v0.1/snapshots/ae9d75c6b4eb39515def78c685fb4d71d49fc2cf/tokenizer.json",
   "trust_remote_code": false,
   "unk_token": "<unk>",
   "use_default_system_prompt": true,
-  "use_fast": true
 }

   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
+      "lstrip": false,
       "normalized": false,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "1": {
       "content": "<s>",
+      "lstrip": false,
       "normalized": false,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "2": {
       "content": "</s>",
+      "lstrip": false,
       "normalized": false,
+      "rstrip": false,
       "single_word": false,
       "special": true
     }
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "trust_remote_code": false,
   "unk_token": "<unk>",
   "use_default_system_prompt": true,
 }