fined tuned on alt-text-validation

README.md

@@ -68,8 +68,12 @@ You can get that checkpoint using the 3083a3cef6e3c8dd90df3f088074bbe836b0f403 c
 
 It was then further fine-tuned on :
 
-- Flickr30k debiased: https://huggingface.co/datasets/Mozilla/flickr30k-transformed-captions
-- DocOrNot: https://huggingface.co/datasets/Mozilla/docornot
+- [Flickr30k debiased](https://huggingface.co/datasets/Mozilla/flickr30k-transformed-captions)
+- [DocOrNot](https://huggingface.co/datasets/Mozilla/docornot)
+- [Alt Text Validation](https://huggingface.co/datasets/Mozilla/alt-text-validation)
+
+For the latter, the dataset was annotated by our team to correct the alt text generayed by the model,
+using the [checkvite tool](https://github.com/mozila/checkvite).
 
 You can find the code used to create the model here: https://github.com/mozilla/distilvit
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "mozilla/distilvit",
+  "_name_or_path": "/Volumes/SSD/vit-base-patch16-224-distilgpt2",
   "architectures": [
     "VisionEncoderDecoderModel"
   ],

generation_config.json

@@ -1,6 +1,5 @@
 {
   "bos_token_id": 50256,
-  "do_sample": false,
   "early_stopping": true,
   "eos_token_id": 50256,
   "max_length": 50,
@@ -8,8 +7,6 @@
   "num_beams": 2,
   "pad_token_id": 50256,
   "repetition_penalty": 1.2,
-  "transformers_version": "4.33.2",
   "seed": 12,
-  "use_cache": true
+  "transformers_version": "4.33.2"
 }
-

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eb10c7e6caaf59d39c22b873daa806f504ea670c4757dbd67a8b5e3be97be6b3
+oid sha256:74574a184acd40eb36ffb93d03ea3780bbe8f8f5d019aabbdf7554304e704a05
 size 729979160

onnx/decoder_model.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7351a7eaeeec455e70e68007a1dcf7307e245473f53da6b13312016ad5d992d3
+oid sha256:e3704718aee4c999920574a3fb45b959ee59a9eb687bc5ff39c1ec6c3b3d32fc
 size 385864797

onnx/decoder_model_merged.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f951eec84e5b043b03ac6a94556e7b0f16846282bc577adec061c848119aadac
+oid sha256:dfc0cac17407a66735b4b4dd5ce62900967a015a618f24c6036b628e1c600328
 size 387342586

onnx/decoder_model_merged_quantized.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1ff8ee2b032955fd71b217c61751a92a6386e7b0e31228ef806a89a923bca34d
-size 99759578
+oid sha256:083cc1aa2c10ee51436601f996091d4633ff2ad6ab933f1197e23c14893fc89e
+size 99759579

onnx/decoder_model_quantized.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:737759dec1b90e1e29eb61e271951db84e3e677d6f8332db64bd4a15fe931711
-size 98065762
+oid sha256:230dd40ee4ba5bd9ec09ad783b45d98ca8acd7b37ff1f73ddbfa6a63c63163ba
+size 98065763

onnx/decoder_with_past_model.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b99196af93603df1ef7eeb9676177d99c0a4a11b01e266ae5cb9327abe968c54
+oid sha256:aae44e11c82e59d84ac7c9f654942dfbcbb2c0d2a03bc8ec49876e81dc3393a1
 size 385864377

onnx/decoder_with_past_model_quantized.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:12e5aad5d80e36cd4f9b2d45297732b01fd9e6503e6393aa393b4c13533c53e7
-size 98063169
+oid sha256:b1823eff0a8dc274eabb3df67551bc7330d4e22a22c23676ff978eb33c643896
+size 98063170

onnx/encoder_model.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aacd44f8cc26c1a0539f5db1fcb98cf3baad2c5e62583b913ed501daf1419e96
+oid sha256:ca2910ada0c94d6f7d72c80e7e4f40916c0bfbd1cff763ad86d96e02fd527e90
 size 343440632

onnx/encoder_model_quantized.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:13062fd1801c38f1039ed439e792b9e08dc8eb0b006d4dd64146575e641c4286
+oid sha256:b9930375f2182d3cb829d78750939633587891b18edb0a66eeb569317ccc0776
 size 87038173

quantize_config.json

@@ -2,122 +2,122 @@
     "per_channel": false,
     "reduce_range": false,
     "per_model_config": {
-        "decoder_with_past_model": {
+        "encoder_model": {
             "op_types": [
-                "Sqrt",
-                "Split",
-                "Unsqueeze",
-                "Tanh",
-                "Gemm",
                 "MatMul",
-                "Cast",
-                "Div",
-                "Pow",
                 "Add",
-                "Constant",
-                "Reshape",
+                "Equal",
                 "ConstantOfShape",
-                "Range",
-                "Slice",
+                "Pow",
+                "Erf",
+                "Sub",
+                "Mul",
+                "Reshape",
+                "Div",
                 "Where",
-                "Concat",
+                "Unsqueeze",
+                "Expand",
+                "Gather",
+                "Softmax",
+                "Transpose",
+                "Constant",
                 "Shape",
-                "Squeeze",
                 "ReduceMean",
-                "Softmax",
-                "Gather",
-                "Sub",
-                "Mul",
-                "Transpose"
+                "Concat",
+                "Sqrt",
+                "Slice",
+                "Conv"
             ],
-            "weight_type": "QInt8"
+            "weight_type": "QUInt8"
         },
         "decoder_model": {
             "op_types": [
-                "Sqrt",
-                "Split",
-                "Unsqueeze",
-                "Tanh",
-                "Gemm",
                 "MatMul",
-                "Cast",
-                "Div",
-                "Pow",
                 "Add",
-                "Constant",
-                "Reshape",
                 "ConstantOfShape",
-                "Range",
-                "Slice",
+                "Pow",
+                "Sub",
+                "Mul",
+                "Reshape",
+                "Div",
                 "Where",
-                "Concat",
-                "Shape",
+                "Tanh",
+                "Unsqueeze",
+                "Gather",
+                "Softmax",
+                "Split",
+                "Slice",
                 "Squeeze",
+                "Range",
+                "Transpose",
+                "Cast",
+                "Constant",
+                "Shape",
                 "ReduceMean",
-                "Softmax",
-                "Gather",
-                "Sub",
-                "Mul",
-                "Transpose"
+                "Concat",
+                "Sqrt",
+                "Gemm"
             ],
             "weight_type": "QInt8"
         },
-        "encoder_model": {
+        "decoder_with_past_model": {
             "op_types": [
-                "Sqrt",
-                "Unsqueeze",
-                "Equal",
                 "MatMul",
-                "Div",
-                "Pow",
                 "Add",
-                "Erf",
-                "Constant",
-                "Reshape",
                 "ConstantOfShape",
-                "Slice",
+                "Pow",
+                "Sub",
+                "Mul",
+                "Reshape",
+                "Div",
                 "Where",
-                "Conv",
-                "Concat",
+                "Tanh",
+                "Unsqueeze",
+                "Gather",
+                "Softmax",
+                "Split",
+                "Slice",
+                "Squeeze",
+                "Range",
+                "Transpose",
+                "Cast",
+                "Constant",
                 "Shape",
                 "ReduceMean",
-                "Softmax",
-                "Gather",
-                "Expand",
-                "Mul",
-                "Sub",
-                "Transpose"
+                "Concat",
+                "Sqrt",
+                "Gemm"
             ],
-            "weight_type": "QUInt8"
+            "weight_type": "QInt8"
         },
         "decoder_model_merged": {
             "op_types": [
-                "Sqrt",
-                "Split",
-                "If",
-                "Unsqueeze",
-                "Tanh",
-                "Gemm",
                 "MatMul",
-                "Cast",
-                "Div",
-                "Pow",
                 "Add",
-                "Constant",
-                "Reshape",
                 "ConstantOfShape",
-                "Range",
-                "Slice",
+                "Pow",
+                "Sub",
+                "Mul",
+                "Gemm",
+                "Reshape",
+                "Div",
                 "Where",
-                "Concat",
-                "Shape",
+                "Tanh",
+                "Unsqueeze",
+                "Gather",
+                "Softmax",
+                "Split",
                 "Squeeze",
+                "Range",
+                "Transpose",
+                "Cast",
+                "Constant",
+                "If",
+                "Shape",
                 "ReduceMean",
-                "Softmax",
-                "Gather",
-                "Sub",
-                "Mul",
-                "Transpose"
+                "Concat",
+                "Sqrt",
+                "Slice"
             ],
             "weight_type": "QInt8"
         }

special_tokens_map.json

@@ -1,30 +1,6 @@
 {
-  "bos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  }
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
 }

tokenizer.json

@@ -9,7 +9,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": true,
+      "normalized": false,
       "special": true
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d6dbda62eac800cee527380ece0a33f7df91a801dc0529c3c35871cb1123276
+oid sha256:ac92771c5176fd55689bb44c37e3467ce9f70d1a9d642ce3f6e6e4e449de33cd
 size 5176