Upload processor (#4)

- Upload processor (5451064815e2cd3556354241f2f4a45c7c658cf0)


Co-authored-by: Pablo Montalvo <[email protected]>

preprocessor_config.json

@@ -0,0 +1,8 @@
+{
+  "image_processor_type": "FuyuImageProcessor",
+  "padding_mode": "constant",
+  "padding_value": 1.0,
+  "processor_class": "FuyuProcessor",
+  "target_height": 1080,
+  "target_width": 1920
+}

special_tokens_map.json

@@ -1,4 +1,8 @@
 {
+  "additional_special_tokens": [
+    "<unk>",
+    "|ENDOFTEXT|"
+  ],
   "bos_token": "|ENDOFTEXT|",
   "eos_token": "|ENDOFTEXT|",
   "unk_token": "<unk>"

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c624be2d9dddae3e8c82cda48fe5c737ec6710225e953ef851a4023f2d4796a
-size 16464051
+oid sha256:20119416a890c3fefc86210999fb8189dde194c12f7f861dbb3bc7a922baaabb
+size 16454569

tokenizer_config.json

@@ -1,11 +1,9 @@
 {
-  "add_bos_token": true,
-  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
@@ -13,19 +11,23 @@
     "71013": {
       "content": "|ENDOFTEXT|",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     }
   },
-  "additional_special_tokens": [],
+  "additional_special_tokens": [
+    "<unk>",
+    "|ENDOFTEXT|"
+  ],
   "bos_token": "|ENDOFTEXT|",
   "clean_up_tokenization_spaces": false,
   "eos_token": "|ENDOFTEXT|",
   "legacy": true,
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": null,
+  "processor_class": "FuyuProcessor",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",