Upload 5 files

tokenizer_2/merges.txt

@@ -48892,4 +48892,4 @@ si gue</w>
 fun ko
 rend ous</w>
 ðŁĴ ij</w>
-jeky ll</w>
+jeky ll</w>

tokenizer_2/special_tokens_map.json

@@ -1,30 +1 @@
-{
-  "bos_token": {
-    "content": "<|startoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "!",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}
+{"bos_token": {"content": "<|startoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "pad_token": "<|endoftext|>"}

tokenizer_2/tokenizer_config.json

@@ -1,38 +1,34 @@
 {
-    "add_prefix_space": false,
-    "added_tokens_decoder": {
-        "0": {
-            "content": "!",
-            "lstrip": false,
-            "normalized": false,
-            "rstrip": false,
-            "single_word": false,
-            "special": true
-        },
-        "49406": {
-            "content": "<|startoftext|>",
-            "lstrip": false,
-            "normalized": true,
-            "rstrip": false,
-            "single_word": false,
-            "special": true
-        },
-        "49407": {
-            "content": "<|endoftext|>",
-            "lstrip": false,
-            "normalized": false,
-            "rstrip": false,
-            "single_word": false,
-            "special": true
-        }
+    "unk_token": {
+        "content": "<|endoftext|>",
+        "single_word": false,
+        "lstrip": false,
+        "rstrip": false,
+        "normalized": true,
+        "__type": "AddedToken"
     },
-    "bos_token": "<|startoftext|>",
-    "clean_up_tokenization_spaces": true,
-    "do_lower_case": true,
-    "eos_token": "<|endoftext|>",
+    "bos_token": {
+        "content": "<|startoftext|>",
+        "single_word": false,
+        "lstrip": false,
+        "rstrip": false,
+        "normalized": true,
+        "__type": "AddedToken"
+    },
+    "eos_token": {
+        "content": "<|endoftext|>",
+        "single_word": false,
+        "lstrip": false,
+        "rstrip": false,
+        "normalized": true,
+        "__type": "AddedToken"
+    },
+    "pad_token": "<|endoftext|>",
+    "add_prefix_space": false,
     "errors": "replace",
+    "do_lower_case": true,
+    "name_or_path": "openai/clip-vit-base-patch32",
     "model_max_length": 77,
-    "pad_token": "!",
-    "tokenizer_class": "CLIPTokenizerFast",
-    "unk_token": "<|endoftext|>"
+    "special_tokens_map_file": "./special_tokens_map.json",
+    "tokenizer_class": "CLIPTokenizer"
 }