2022 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license this function prepares a list of pil images numimages 2 images np random randint255 size3 30 400 dtypenp uint8 for x in rangenumimages images image fromarraynp moveaxisx 0 1 for x in images printtypex for x in images die prompts text and 1 image user images0 describe this image nassistant text and images user images0 describe this image nassistant an image of two dogs n user images1 describe this image nassistant only text user describe this image nassistant an image of two kittens n user describe this image nassistant only images images0 images1 return prompts def testsaveloadpretrainedadditionalfeaturesself processor ideficsprocessortokenizerself gettokenizer imageprocessorself getimageprocessor processor savepretrainedself checkpointpath tokenizeraddkwargs self gettokenizerbostokenbos eostokeneos imageprocessoraddkwargs self getimageprocessordonormalizefalse paddingvalue1 0 processor ideficsprocessor frompretrained self checkpointpath bostokenbos eostokeneos donormalizefalse paddingvalue1 0 self assertequalprocessor tokenizer getvocab tokenizeraddkwargs getvocab self assertisinstanceprocessor tokenizer pretrainedtokenizerfast self assertequalprocessor imageprocessor tojsonstring imageprocessoraddkwargs tojsonstring self assertisinstanceprocessor imageprocessor ideficsimageprocessor def testprocessorself imageprocessor self getimageprocessor tokenizer self gettokenizer processor ideficsprocessortokenizertokenizer imageprocessorimageprocessor prompts self prepareprompts test that all prompts succeeded inputprocessor processorprompts returntensorspt for key in self inputkeys assert torch istensorinputprocessorkey def testtokenizerdecodeself imageprocessor self getimageprocessor tokenizer self gettokenizer processor ideficsprocessortokenizertokenizer imageprocessorimageprocessor predictedids 1 4 5 8 1 0 8 3 4 3 1 1 8 9 decodedprocessor processor batchdecodepredictedids decodedtok tokenizer batchdecodepredictedids self assertlistequaldecodedtok decodedprocessor def testtokenizerpaddingself imageprocessor self getimageprocessor tokenizer self gettokenizerpaddingsideright processor ideficsprocessortokenizertokenizer imageprocessorimageprocessor predictedtokens s describe this image nassistant unkunkunkunkunkunkunkunkunk s describe this image nassistant unkunkunkunkunkunkunkunkunkunk prompts prompt for prompt in self prepareprompts2 maxlength processorprompts paddingmaxlength truncationtrue maxlength20 longest processorprompts paddinglongest truncationtrue maxlength30 decodedmaxlength processor tokenizer decodemaxlengthinputids1 decodedlongest processor tokenizer decodelongestinputids1 self assertequaldecodedmaxlength predictedtokens1 self assertequaldecodedlongest predictedtokens0 def testmodelinputnamesself imageprocessor self getimageprocessor tokenizer self gettokenizer processor ideficsprocessortokenizertokenizer imageprocessorimageprocessor prompts self prepareprompts inputs processorprompts for now the processor supports only pixelvalues inputids attentionmask self assertsetequalsetinputs keys setself inputkeys 2022 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license this function prepares a list of pil images print type x for x in images die text and 1 image text and images only text only images test that all prompts succeeded for now the processor supports only pixel_values input_ids attention_mask
import numpy as np

from transformers.testing_utils import TestCasePlus, require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available


if is_torch_available():
    import torch

if is_vision_available():
    from PIL import Image

    from transformers import (
        AutoProcessor,
        IdeficsImageProcessor,
        IdeficsProcessor,
        LlamaTokenizerFast,
        PreTrainedTokenizerFast,
    )


@require_torch
@require_vision
class IdeficsProcessorTest(TestCasePlus):
    def setUp(self):
        super().setUp()

        self.checkpoint_path = self.get_auto_remove_tmp_dir()

        image_processor = IdeficsImageProcessor()
        tokenizer = LlamaTokenizerFast.from_pretrained("HuggingFaceM4/tiny-random-idefics")

        processor = IdeficsProcessor(image_processor, tokenizer)

        processor.save_pretrained(self.checkpoint_path)

        self.input_keys = ["pixel_values", "input_ids", "attention_mask", "image_attention_mask"]

    def get_tokenizer(self, **kwargs):
        return AutoProcessor.from_pretrained(self.checkpoint_path, **kwargs).tokenizer

    def get_image_processor(self, **kwargs):
        return AutoProcessor.from_pretrained(self.checkpoint_path, **kwargs).image_processor

    def prepare_prompts(self):
        

        num_images = 2
        images = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8) for x in range(num_images)]
        images = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in images]

        
        

        prompts = [
            
            [
                "User:",
                images[0],
                "Describe this image.\nAssistant:",
            ],
            
            [
                "User:",
                images[0],
                "Describe this image.\nAssistant: An image of two dogs.\n",
                "User:",
                images[1],
                "Describe this image.\nAssistant:",
            ],
            
            [
                "User:",
                "Describe this image.\nAssistant: An image of two kittens.\n",
                "User:",
                "Describe this image.\nAssistant:",
            ],
            
            [
                images[0],
                images[1],
            ],
        ]

        return prompts

    def test_save_load_pretrained_additional_features(self):
        processor = IdeficsProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
        processor.save_pretrained(self.checkpoint_path)

        tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
        image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)

        processor = IdeficsProcessor.from_pretrained(
            self.checkpoint_path, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
        )

        self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
        self.assertIsInstance(processor.tokenizer, PreTrainedTokenizerFast)

        self.assertEqual(processor.image_processor.to_json_string(), image_processor_add_kwargs.to_json_string())
        self.assertIsInstance(processor.image_processor, IdeficsImageProcessor)

    def test_processor(self):
        image_processor = self.get_image_processor()
        tokenizer = self.get_tokenizer()

        processor = IdeficsProcessor(tokenizer=tokenizer, image_processor=image_processor)

        prompts = self.prepare_prompts()

        
        input_processor = processor(prompts, return_tensors="pt")
        for key in self.input_keys:
            assert torch.is_tensor(input_processor[key])

    def test_tokenizer_decode(self):
        image_processor = self.get_image_processor()
        tokenizer = self.get_tokenizer()

        processor = IdeficsProcessor(tokenizer=tokenizer, image_processor=image_processor)

        predicted_ids = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

        decoded_processor = processor.batch_decode(predicted_ids)
        decoded_tok = tokenizer.batch_decode(predicted_ids)

        self.assertListEqual(decoded_tok, decoded_processor)

    def test_tokenizer_padding(self):
        image_processor = self.get_image_processor()
        tokenizer = self.get_tokenizer(padding_side="right")

        processor = IdeficsProcessor(tokenizer=tokenizer, image_processor=image_processor)

        predicted_tokens = [
            "<s> Describe this image.\nAssistant:<unk><unk><unk><unk><unk><unk><unk><unk><unk>",
            "<s> Describe this image.\nAssistant:<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk>",
        ]
        prompts = [[prompt] for prompt in self.prepare_prompts()[2]]
        max_length = processor(prompts, padding="max_length", truncation=True, max_length=20)
        longest = processor(prompts, padding="longest", truncation=True, max_length=30)
        decoded_max_length = processor.tokenizer.decode(max_length["input_ids"][-1])
        decoded_longest = processor.tokenizer.decode(longest["input_ids"][-1])
        self.assertEqual(decoded_max_length, predicted_tokens[1])
        self.assertEqual(decoded_longest, predicted_tokens[0])

    def test_model_input_names(self):
        image_processor = self.get_image_processor()
        tokenizer = self.get_tokenizer()

        processor = IdeficsProcessor(tokenizer=tokenizer, image_processor=image_processor)
        prompts = self.prepare_prompts()

        inputs = processor(prompts)

        
        self.assertSetEqual(set(inputs.keys()), set(self.input_keys))