Upload 6 files

.gitattributes

@@ -2,27 +2,20 @@
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
 *.ftz filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
@@ -30,5 +23,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,61 @@
 ---
 license: apache-2.0
 ---
-OFA-huge checkpoint optimized for image-captioning 
 
-It is transformed from fairseq to huggingface and no other processing has been performed.
+# OFA-huge
+This is the **huge** version of OFA pretrained model. OFA is a unified multimodal pretrained model that unifies modalities (i.e., cross-modality, vision, language) and tasks (e.g., image generation, visual grounding, image captioning, image classification, text generation, etc.) to a simple sequence-to-sequence learning framework.
 
-This checkpoint is not uploaded by the official OFA-sys, so i upload it for use.
+The directory includes 4 files, namely `config.json` which consists of model configuration, `vocab.json` and `merge.txt` for our OFA tokenizer, and lastly `pytorch_model.bin` which consists of model weights. There is no need to worry about the mismatch between Fairseq and transformers, since we have addressed the issue yet. 
 
-Thanks to OFA-sys
-(https://github.com/OFA-Sys/OFA)
-(https://huggingface.co/OFA-Sys/ofa-huge)
+To use it in transformers, please refer to https://github.com/OFA-Sys/OFA/tree/feature/add_transformers. Install the transformers and download the models as shown below.
+```
+git clone --single-branch --branch feature/add_transformers https://github.com/OFA-Sys/OFA.git
+pip install OFA/transformers/
+git clone https://huggingface.co/OFA-Sys/OFA-huge
+```
+After, refer the path to OFA-huge to `ckpt_dir`, and prepare an image for the testing example below. Also, ensure that you have pillow and torchvision in your environment. 
+
+```
+>>> from PIL import Image
+>>> from torchvision import transforms
+>>> from transformers import OFATokenizer, OFAModel
+>>> from generate import sequence_generator
+
+>>> mean, std = [0.5, 0.5, 0.5], [0.5, 0.5, 0.5]
+>>> resolution = 480
+>>> patch_resize_transform = transforms.Compose([
+        lambda image: image.convert("RGB"),
+        transforms.Resize((resolution, resolution), interpolation=Image.BICUBIC),
+        transforms.ToTensor(), 
+        transforms.Normalize(mean=mean, std=std)
+    ])
+
+
+>>> tokenizer = OFATokenizer.from_pretrained(ckpt_dir)
+
+>>> txt = " what does the image describe?"
+>>> inputs = tokenizer([txt], return_tensors="pt").input_ids
+>>> img = Image.open(path_to_image)
+>>> patch_img = patch_resize_transform(img).unsqueeze(0)
+
+
+>>> # using the generator of fairseq version
+>>> model = OFAModel.from_pretrained(ckpt_dir, use_cache=True)
+>>> generator = sequence_generator.SequenceGenerator(
+                    tokenizer=tokenizer,
+                    beam_size=5,
+                    max_len_b=16, 
+                    min_len=0,
+                    no_repeat_ngram_size=3,
+                )
+>>> data = {}
+>>> data["net_input"] = {"input_ids": inputs, 'patch_images': patch_img, 'patch_masks':torch.tensor([True])}
+>>> gen_output = generator.generate([model], data)
+>>> gen = [gen_output[i][0]["tokens"] for i in range(len(gen_output))]
+
+>>> # using the generator of huggingface version
+>>> model = OFAModel.from_pretrained(ckpt_dir, use_cache=False)
+>>> gen = model.generate(inputs, patch_images=patch_img, num_beams=5, no_repeat_ngram_size=3) 
+
+>>> print(tokenizer.batch_decode(gen, skip_special_tokens=True))
+```

config.json

@@ -0,0 +1,52 @@
+{
+  "activation_dropout": 0.0,
+  "activation_function": "gelu",
+  "add_type_embedding": true,
+  "architectures": [
+    "OFAModel"
+  ],
+  "attention_dropout": 0.0,
+  "attn_scale_factor": 2.0,
+  "bos_token_id": 0,
+  "classifier_dropout": 0.0,
+  "code_image_size": 128,
+  "code_layernorm_embedding": true,
+  "d_model": 1280,
+  "decoder_attention_heads": 16,
+  "decoder_drop_path_rate": 0.0,
+  "decoder_ffn_dim": 5120,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 12,
+  "decoder_normalize_before": true,
+  "decoder_start_token_id": 0,
+  "dropout": 0.1,
+  "encoder_attention_heads": 16,
+  "encoder_drop_path_rate": 0.0,
+  "encoder_ffn_dim": 5120,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 24,
+  "encoder_normalize_before": true,
+  "entangle_position_embedding": false,
+  "eos_token_id": 2,
+  "forced_eos_token_id": 2,
+  "image_bucket_size": 42,
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "layernorm_embedding": true,
+  "max_position_embeddings": 1024,
+  "model_type": "ofa",
+  "normformer": true,
+  "num_hidden_layers": 24,
+  "pad_token_id": 1,
+  "patch_layernorm_embedding": true,
+  "resnet_drop_path_rate": 0.0,
+  "resnet_model_path": null,
+  "resnet_type": "resnet152",
+  "scale_embedding": false,
+  "share_decoder_input_output_embed": true,
+  "token_bucket_size": 256,
+  "torch_dtype": "float32",
+  "transformers_version": "4.15.0",
+  "use_cache": false,
+  "vocab_size": 59457
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dc6d4d382ab2405cc94b3d52ea6c6ce57da83c5f4590511f0bb2c4aa98e0e08
+size 4394603544