codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch fuyu model import io import unittest import requests from transformers import fuyuconfig istorchavailable isvisionavailable from transformers testingutils import requiretorch requiretorchgpu slow torchdevice from transformers utils import cachedproperty from testmodelingcommon import modeltestermixin idstensor randomattentionmask from testpipelinemixin import pipelinetestermixin if isvisionavailable from pil import image if istorchavailable and isvisionavailable from transformers import fuyuprocessor if istorchavailable import torch from transformers import fuyuforcausallm class fuyumodeltester def init self parent batchsize13 seqlength7 imagesize30 patchsize15 numchannels3 istrainingtrue useinputmasktrue uselabelstrue vocabsize99 hiddensize32 numhiddenlayers2 numattentionheads4 intermediatesize37 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 maxpositionembeddings512 typevocabsize16 typesequencelabelsize2 initializerrange0 02 numlabels3 numchoices4 padtokenid0 scopenone self parent parent self batchsize batchsize self seqlength seqlength self imagesize imagesize self patchsize patchsize self numchannels numchannels self istraining istraining self useinputmask useinputmask self uselabels uselabels self vocabsize vocabsize self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self maxpositionembeddings maxpositionembeddings self typevocabsize typevocabsize self typesequencelabelsize typesequencelabelsize self initializerrange initializerrange self numlabels numlabels self numchoices numchoices self padtokenid padtokenid self scope scope def prepareconfigandinputsself inputids idstensorself batchsize self seqlength self vocabsize inputmask none if self useinputmask inputmask randomattentionmaskself batchsize self seqlength sequencelabels none tokenlabels none if self uselabels sequencelabels idstensorself batchsize self typesequencelabelsize tokenlabels idstensorself batchsize self seqlength self numlabels config self getconfig return config inputids inputmask sequencelabels tokenlabels def getconfigself return fuyuconfig vocabsizeself vocabsize hiddensizeself hiddensize numhiddenlayersself numhiddenlayers numattentionheadsself numattentionheads intermediatesizeself intermediatesize hiddenactself hiddenact hiddendropoutprobself hiddendropoutprob attentionprobsdropoutprobself attentionprobsdropoutprob maxpositionembeddingsself maxpositionembeddings typevocabsizeself typevocabsize isdecoderfalse initializerrangeself initializerrange padtokenidself padtokenid def createandcheckmodel self config inputids inputmask sequencelabels tokenlabels model fuyuforcausallmconfigconfig model totorchdevice model eval result modelinputids attentionmaskinputmask result modelinputids self parent assertequalresult lasthiddenstate shape self batchsize self seqlength self hiddensize def createandcheckmodelasdecoder self config inputids inputmask sequencelabels tokenlabels encoderhiddenstates encoderattentionmask config addcrossattention true model fuyuforcausallmconfig model totorchdevice model eval result model inputids attentionmaskinputmask encoderhiddenstatesencoderhiddenstates encoderattentionmaskencoderattentionmask result model inputids attentionmaskinputmask encoderhiddenstatesencoderhiddenstates result modelinputids attentionmaskinputmask self parent assertequalresult lasthiddenstate shape self batchsize self seqlength self hiddensize def createandcheckforcausallm self config inputids inputmask sequencelabels tokenlabels encoderhiddenstates encoderattentionmask model fuyuforcausallmconfigconfig model totorchdevice model eval result modelinputids attentionmaskinputmask labelstokenlabels self parent assertequalresult logits shape self batchsize self seqlength self vocabsize def createandcheckdecodermodelpastlargeinputs self config inputids inputmask sequencelabels tokenlabels encoderhiddenstates encoderattentionmask config isdecoder true config addcrossattention true model fuyuforcausallmconfigconfig model totorchdevice model eval first forward pass outputs model inputids attentionmaskinputmask encoderhiddenstatesencoderhiddenstates encoderattentionmaskencoderattentionmask usecachetrue pastkeyvalues outputs pastkeyvalues create hypothetical multiple next token and extent to nextinputids nexttokens idstensorself batchsize 3 config vocabsize nextmask idstensorself batchsize 3 vocabsize2 append to next inputids and nextinputids torch catinputids nexttokens dim1 nextattentionmask torch catinputmask nextmask dim1 outputfromnopast model nextinputids attentionmasknextattentionmask encoderhiddenstatesencoderhiddenstates encoderattentionmaskencoderattentionmask outputhiddenstatestrue hiddenstates0 outputfrompast model nexttokens attentionmasknextattentionmask encoderhiddenstatesencoderhiddenstates encoderattentionmaskencoderattentionmask pastkeyvaluespastkeyvalues outputhiddenstatestrue hiddenstates0 select random slice randomsliceidx idstensor1 outputfrompast shape1 item outputfromnopastslice outputfromnopast 3 randomsliceidx detach outputfrompastslice outputfrompast randomsliceidx detach self parent asserttrueoutputfrompastslice shape1 nexttokens shape1 test that outputs are equal for slice self parent asserttruetorch allcloseoutputfrompastslice outputfromnopastslice atol1e3 def prepareconfigandinputsforcommonself configandinputs self prepareconfigandinputs config inputids inputmask sequencelabels tokenlabels configandinputs inputsdict inputids inputids attentionmask inputmask return config inputsdict requiretorch class fuyumodeltestmodeltestermixin pipelinetestermixin unittest testcase allmodelclasses fuyuforcausallm if istorchavailable else pipelinemodelmapping textgeneration fuyuforcausallm if istorchavailable else testheadmasking false testpruning false testcpuoffload false testdiskoffload false testmodelparallel false def setupself self modeltester fuyumodeltesterself unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantfalseself pass todo fix me once this model gets more usage unittest skipdoes not work on the tiny model def testdiskoffloadbinself super testdiskoffload todo fix me once this model gets more usage unittest skipdoes not work on the tiny model def testdiskoffloadsafetensorsself super testdiskoffload todo fix me once this model gets more usage unittest skipdoes not work on the tiny model def testmodelparallelismself super testmodelparallelism slow requiretorchgpu class fuyumodelintegrationtestunittest testcase cachedproperty def defaultprocessorself return fuyuprocessor frompretrainedadeptfuyu8b cachedproperty def defaultmodelself return fuyuforcausallm frompretrainedadeptfuyu8b def testgreedygenerationself processor self defaultprocessor model self defaultmodel url https huggingface codatasetshfinternaltestingfixturescaptioningresolvemainbus png image image openio bytesiorequests geturl content textpromptcococaptioning generate a cocostyle caption n inputs processortexttextpromptcococaptioning imagesimage returntensorspt generatedids model generateinputs maxnewtokens10 take the last 8 tokens in order to skip special nx04 characters and decode them generatedtext processor batchdecodegeneratedids 8 skipspecialtokenstrue0 self assertequalgeneratedtext a blue bus parked on the side of a road coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch fuyu model first forward pass create hypothetical multiple next token and extent to next_input_ids append to next input_ids and select random slice test that outputs are equal for slice todo fix me once this model gets more usage todo fix me once this model gets more usage todo fix me once this model gets more usage take the last 8 tokens in order to skip special n x04 characters and decode them slow require_torch_accelerator def test_model_8b_chat_greedy_generation_bus_color self expected_text_completion the bus is blue n endoftext text_prompt_bus_color what color is the bus n model_inputs_bus_color self processor text text_prompt_bus_color images self bus_image_pil generated_tokens self model generate model_inputs_bus_color max_new_tokens 10 text self processor tokenizer batch_decode generated_tokens end_sequence text 0 split x04 1 clean_sequence end_sequence end_sequence find endoftext len endoftext if endoftext in end_sequence else end_sequence self assertequal expected_text_completion clean_sequence slow require_torch_accelerator def test_model_8b_chat_greedy_generation_chart_vqa self expected_text_tokens the life expectancy at birth of male s in 20 18 is 80 7 n endoftext fmt skip expected_text_completion join expected_text_tokens todo make sure the end string matches text_prompt_chart_vqa what is the highest life expectancy at birth of male n chart_image_url https huggingface co datasets hf internal testing fixtures captioning resolve main chart png chart_image_pil image open io bytesio requests get chart_image_url content model_inputs_chart_vqa self processor text text_prompt_chart_vqa images chart_image_pil generated_tokens self model generate model_inputs_chart_vqa max_new_tokens 10 text self processor tokenizer batch_decode generated_tokens end_sequence text 0 split x04 1 clean_sequence end_sequence end_sequence find endoftext len endoftext if endoftext in end_sequence else end_sequence self assertequal expected_text_completion clean_sequence slow require_torch_accelerator def test_model_8b_chat_greedy_generation_bounding_box self expected_text_completion x00194213202244 x01 endoftext text_prompt_bbox when presented with a box perform ocr to extract text contained within it if provided with text generate the corresponding bounding box nwilliams noqa e231 bbox_image_url https huggingface co datasets hf internal testing fixtures captioning resolve main bbox_sample_image png bbox_image_pil image open io bytesio requests get bbox_image_url content model_inputs_bbox self processor text text_prompt_bbox images bbox_image_pil generated_tokens self model generate model_inputs_bbox max_new_tokens 10 text self processor tokenizer batch_decode generated_tokens end_sequence text 0 split x04 1 clean_sequence end_sequence end_sequence find endoftext len endoftext if endoftext in end_sequence else end_sequence self assertequal expected_text_completion clean_sequence
import io
import unittest

import requests

from transformers import FuyuConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device
from transformers.utils import cached_property

from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_vision_available():
    from PIL import Image


if is_torch_available() and is_vision_available():
    from transformers import FuyuProcessor


if is_torch_available():
    import torch

    from transformers import FuyuForCausalLM


class FuyuModelTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        seq_length=7,
        image_size=30,
        patch_size=15,
        num_channels=3,
        is_training=True,
        use_input_mask=True,
        use_labels=True,
        vocab_size=99,
        hidden_size=32,
        num_hidden_layers=2,
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        type_vocab_size=16,
        type_sequence_label_size=2,
        initializer_range=0.02,
        num_labels=3,
        num_choices=4,
        pad_token_id=0,
        scope=None,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.seq_length = seq_length
        self.image_size = image_size
        self.patch_size = patch_size
        self.num_channels = num_channels
        self.is_training = is_training
        self.use_input_mask = use_input_mask
        self.use_labels = use_labels
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.type_vocab_size = type_vocab_size
        self.type_sequence_label_size = type_sequence_label_size
        self.initializer_range = initializer_range
        self.num_labels = num_labels
        self.num_choices = num_choices
        self.pad_token_id = pad_token_id
        self.scope = scope

    def prepare_config_and_inputs(self):
        input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)

        input_mask = None
        if self.use_input_mask:
            input_mask = random_attention_mask([self.batch_size, self.seq_length])

        sequence_labels = None
        token_labels = None
        if self.use_labels:
            sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
            token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)

        config = self.get_config()

        return config, input_ids, input_mask, sequence_labels, token_labels

    def get_config(self):
        return FuyuConfig(
            vocab_size=self.vocab_size,
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            max_position_embeddings=self.max_position_embeddings,
            type_vocab_size=self.type_vocab_size,
            is_decoder=False,
            initializer_range=self.initializer_range,
            pad_token_id=self.pad_token_id,
        )

    def create_and_check_model(
        self,
        config,
        input_ids,
        input_mask,
        sequence_labels,
        token_labels,
    ):
        model = FuyuForCausalLM(config=config)
        model.to(torch_device)
        model.eval()
        result = model(input_ids, attention_mask=input_mask)
        result = model(input_ids)
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))

    def create_and_check_model_as_decoder(
        self,
        config,
        input_ids,
        input_mask,
        sequence_labels,
        token_labels,
        encoder_hidden_states,
        encoder_attention_mask,
    ):
        config.add_cross_attention = True
        model = FuyuForCausalLM(config)
        model.to(torch_device)
        model.eval()
        result = model(
            input_ids,
            attention_mask=input_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
        )
        result = model(
            input_ids,
            attention_mask=input_mask,
            encoder_hidden_states=encoder_hidden_states,
        )
        result = model(input_ids, attention_mask=input_mask)
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))

    def create_and_check_for_causal_lm(
        self,
        config,
        input_ids,
        input_mask,
        sequence_labels,
        token_labels,
        encoder_hidden_states,
        encoder_attention_mask,
    ):
        model = FuyuForCausalLM(config=config)
        model.to(torch_device)
        model.eval()
        result = model(input_ids, attention_mask=input_mask, labels=token_labels)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))

    def create_and_check_decoder_model_past_large_inputs(
        self,
        config,
        input_ids,
        input_mask,
        sequence_labels,
        token_labels,
        encoder_hidden_states,
        encoder_attention_mask,
    ):
        config.is_decoder = True
        config.add_cross_attention = True
        model = FuyuForCausalLM(config=config)
        model.to(torch_device)
        model.eval()

        
        outputs = model(
            input_ids,
            attention_mask=input_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
            use_cache=True,
        )
        past_key_values = outputs.past_key_values

        
        next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size)
        next_mask = ids_tensor((self.batch_size, 3), vocab_size=2)

        
        next_input_ids = torch.cat([input_ids, next_tokens], dim=-1)
        next_attention_mask = torch.cat([input_mask, next_mask], dim=-1)

        output_from_no_past = model(
            next_input_ids,
            attention_mask=next_attention_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
            output_hidden_states=True,
        )["hidden_states"][0]
        output_from_past = model(
            next_tokens,
            attention_mask=next_attention_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            output_hidden_states=True,
        )["hidden_states"][0]

        
        random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item()
        output_from_no_past_slice = output_from_no_past[:, -3:, random_slice_idx].detach()
        output_from_past_slice = output_from_past[:, :, random_slice_idx].detach()

        self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

        
        self.parent.assertTrue(torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3))

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        (
            config,
            input_ids,
            input_mask,
            sequence_labels,
            token_labels,
        ) = config_and_inputs
        inputs_dict = {"input_ids": input_ids, "attention_mask": input_mask}
        return config, inputs_dict


@require_torch
class FuyuModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
    all_model_classes = (FuyuForCausalLM,) if is_torch_available() else ()
    pipeline_model_mapping = {"text-generation": FuyuForCausalLM} if is_torch_available() else {}

    test_head_masking = False
    test_pruning = False
    test_cpu_offload = False
    test_disk_offload = False
    test_model_parallel = False

    def setUp(self):
        self.model_tester = FuyuModelTester(self)

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant_false(self):
        pass

    
    @unittest.skip("Does not work on the tiny model.")
    def test_disk_offload_bin(self):
        super().test_disk_offload()

    
    @unittest.skip("Does not work on the tiny model.")
    def test_disk_offload_safetensors(self):
        super().test_disk_offload()

    
    @unittest.skip("Does not work on the tiny model.")
    def test_model_parallelism(self):
        super().test_model_parallelism()


@slow
@require_torch_gpu
class FuyuModelIntegrationTest(unittest.TestCase):
    @cached_property
    def default_processor(self):
        return FuyuProcessor.from_pretrained("adept/fuyu-8b")

    @cached_property
    def default_model(self):
        return FuyuForCausalLM.from_pretrained("adept/fuyu-8b")

    def test_greedy_generation(self):
        processor = self.default_processor
        model = self.default_model

        url = "https://huggingface.co/datasets/hf-internal-testing/fixtures-captioning/resolve/main/bus.png"
        image = Image.open(io.BytesIO(requests.get(url).content))

        text_prompt_coco_captioning = "Generate a coco-style caption.\n"

        inputs = processor(text=text_prompt_coco_captioning, images=image, return_tensors="pt")
        generated_ids = model.generate(**inputs, max_new_tokens=10)

        
        generated_text = processor.batch_decode(generated_ids[:, -8:], skip_special_tokens=True)[0]
        self.assertEqual(generated_text, "A blue bus parked on the side of a road.")