"""simple docstring"""
import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class    __snake_case       (     __lowerCAmelCase       ,      unittest.TestCase				):
      a__        =		None
      a__        =		BloomTokenizerFast
      a__        =		BloomTokenizerFast
      a__        =		True
      a__        =		False
      a__        =		"""tokenizer_file"""
      a__        =		{"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}



      def   lowerCamelCase_   (						self)	->			List[str]:





        '''simple docstring'''





        super().setUp()
        a__:     int										=	BloomTokenizerFast.from_pretrained('bigscience/tokenizer')
        tokenizer.save_pretrained(self.tmpdirname)



      def   lowerCamelCase_   (						self    ,			**lowercase)	->			Optional[int]:





        '''simple docstring'''





        kwargs.update(self.special_tokens_map)
        return BloomTokenizerFast.from_pretrained(self.tmpdirname    ,			**lowercase)



      def   lowerCamelCase_   (						self)	->			List[Any]:





        '''simple docstring'''





        a__:     Union[str, Any]										=	self.get_rust_tokenizer()

        a__:     List[str]										=	['The quick brown fox</s>', 'jumps over the lazy dog</s>']
        a__:     List[Any]										=	[[21_75, 2_37_14, 7_31_73, 14_42_52, 2], [77, 13_26_19, 34_78, 3_68, 10_95_86, 3_54_33, 2]]

        a__:     Any										=	tokenizer.batch_encode_plus(lowercase)['input_ids']
        self.assertListEqual(lowercase    ,			lowercase)

        a__:     int										=	tokenizer.batch_decode(lowercase)
        self.assertListEqual(lowercase    ,			lowercase)



      def   lowerCamelCase_   (						self    ,			lowercase=6)	->			Tuple:





        '''simple docstring'''





        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'):
            a__:     Union[str, Any]										=	self.rust_tokenizer_class.from_pretrained(lowercase    ,			**lowercase)
            # tokenizer_r.pad_token = None # Hotfixing padding = None
            # Simple input
            a__:     Dict										=	'This is a simple input'
            a__:     Union[str, Any]										=	['This is a simple input 1', 'This is a simple input 2']
            a__:     Optional[Any]										=	('This is a simple input', 'This is a pair')
            a__:     str										=	[
                ('This is a simple input 1', 'This is a simple input 2'),
                ('This is a simple pair 1', 'This is a simple pair 2'),
            ]

            # Simple input tests
            try:
              tokenizer_r.encode(lowercase    ,			max_length=lowercase)
              tokenizer_r.encode_plus(lowercase    ,			max_length=lowercase)

              tokenizer_r.batch_encode_plus(lowercase    ,			max_length=lowercase)
              tokenizer_r.encode(lowercase    ,			max_length=lowercase)
              tokenizer_r.batch_encode_plus(lowercase    ,			max_length=lowercase)
            except ValueError:
              self.fail('Bloom Tokenizer should be able to deal with padding')

            a__:     Optional[int]										=	None  # Hotfixing padding = None
            self.assertRaises(lowercase    ,			tokenizer_r.encode    ,			lowercase    ,			max_length=lowercase    ,			padding='max_length')

            # Simple input
            self.assertRaises(lowercase    ,			tokenizer_r.encode_plus    ,			lowercase    ,			max_length=lowercase    ,			padding='max_length')

            # Simple input
            self.assertRaises(
                lowercase    ,			tokenizer_r.batch_encode_plus    ,			lowercase    ,			max_length=lowercase    ,			padding='max_length'    ,			)

            # Pair input
            self.assertRaises(lowercase    ,			tokenizer_r.encode    ,			lowercase    ,			max_length=lowercase    ,			padding='max_length')

            # Pair input
            self.assertRaises(lowercase    ,			tokenizer_r.encode_plus    ,			lowercase    ,			max_length=lowercase    ,			padding='max_length')

            # Pair input
            self.assertRaises(
                lowercase    ,			tokenizer_r.batch_encode_plus    ,			lowercase    ,			max_length=lowercase    ,			padding='max_length'    ,			)



      def   lowerCamelCase_   (						self)	->			List[str]:





        '''simple docstring'''





        a__:     List[Any]										=	self.get_rust_tokenizer()
        a__:     Optional[Any]										=	load_dataset('xnli'    ,			'all_languages'    ,			split='test'    ,			streaming=lowercase)

        a__:     Any										=	next(iter(lowercase))['premise']  # pick up one data
        a__:     List[str]										=	list(sample_data.values())

        a__:     List[str]										=	list(map(tokenizer.encode    ,			lowercase))
        a__:     Union[str, Any]										=	[tokenizer.decode(lowercase    ,			clean_up_tokenization_spaces=lowercase) for x in output_tokens]
        self.assertListEqual(lowercase    ,			lowercase)



      def   lowerCamelCase_   (						self)	->			List[str]:





        '''simple docstring'''





        self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map)    ,			1)
        self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0])    ,			1)



"""simple docstring"""
import shutil
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import (
    is_pt_tf_cross_test,
    require_tf,
    require_torch,
    require_torchvision,
    require_vision,
)
from transformers.utils import is_tf_available, is_torch_available, is_vision_available


if is_vision_available():
 from PIL import Image

 from transformers import AutoProcessor, SamImageProcessor, SamProcessor

if is_torch_available():
 import torch

if is_tf_available():
 import tensorflow as tf




@require_vision
@require_torchvision
class    __snake_case       (     unittest.TestCase				):



      def   lowerCamelCase_   (						self)	->			Any:





        '''simple docstring'''





        a__:     Optional[Any]										=	tempfile.mkdtemp()
        a__:     Optional[int]										=	SamImageProcessor()
        a__:     Tuple										=	SamProcessor(lowercase)
        processor.save_pretrained(self.tmpdirname)



      def   lowerCamelCase_   (						self    ,			**lowercase)	->			List[Any]:





        '''simple docstring'''





        return AutoProcessor.from_pretrained(self.tmpdirname    ,			**lowercase).image_processor



      def   lowerCamelCase_   (						self)	->			Optional[int]:





        '''simple docstring'''





        shutil.rmtree(self.tmpdirname)



      def   lowerCamelCase_   (						self)	->			int:





        '''simple docstring'''





        a__:     Any										=	[np.random.randint(2_55    ,			size=(3, 30, 4_00)    ,			dtype=np.uinta)]

        a__:     Optional[Any]										=	[Image.fromarray(np.moveaxis(lowercase    ,			0    ,			-1)) for x in image_inputs]

        return image_inputs



      def   lowerCamelCase_   (						self)	->			str:





        '''simple docstring'''





        a__:     List[str]										=	SamProcessor(image_processor=self.get_image_processor())
        processor.save_pretrained(self.tmpdirname)

        a__:     Optional[int]										=	self.get_image_processor(do_normalize=lowercase    ,			padding_value=1.0)

        a__:     List[Any]										=	SamProcessor.from_pretrained(self.tmpdirname    ,			do_normalize=lowercase    ,			padding_value=1.0)

        self.assertEqual(processor.image_processor.to_json_string()    ,			image_processor_add_kwargs.to_json_string())
        self.assertIsInstance(processor.image_processor    ,			lowercase)



      def   lowerCamelCase_   (						self)	->			Tuple:





        '''simple docstring'''





        a__:     Union[str, Any]										=	self.get_image_processor()

        a__:     List[Any]										=	SamProcessor(image_processor=lowercase)

        a__:     Optional[int]										=	self.prepare_image_inputs()

        a__:     Optional[Any]										=	image_processor(lowercase    ,			return_tensors='np')
        a__:     Tuple										=	processor(images=lowercase    ,			return_tensors='np')

        input_feat_extract.pop('original_sizes')  # pop original_sizes as it is popped in the processor
        input_feat_extract.pop('reshaped_input_sizes')  # pop original_sizes as it is popped in the processor

        for key in input_feat_extract.keys():
          self.assertAlmostEqual(input_feat_extract[key].sum()    ,			input_processor[key].sum()    ,			delta=1e-2)



      @require_torch
      def   lowerCamelCase_   (						self)	->			Any:





        '''simple docstring'''





        a__:     int										=	self.get_image_processor()

        a__:     List[str]										=	SamProcessor(image_processor=lowercase)
        a__:     Optional[Any]										=	[torch.ones((1, 3, 5, 5))]

        a__:     Union[str, Any]										=	[[17_64, 26_46]]

        a__:     Optional[Any]										=	[[6_83, 10_24]]
        a__:     int										=	processor.post_process_masks(lowercase    ,			lowercase    ,			lowercase)
        self.assertEqual(masks[0].shape    ,			(1, 3, 17_64, 26_46))

        a__:     Optional[int]										=	processor.post_process_masks(
            lowercase    ,			torch.tensor(lowercase)    ,			torch.tensor(lowercase))
        self.assertEqual(masks[0].shape    ,			(1, 3, 17_64, 26_46))

        # should also work with np
        a__:     Dict										=	[np.ones((1, 3, 5, 5))]
        a__:     Tuple										=	processor.post_process_masks(lowercase    ,			np.array(lowercase)    ,			np.array(lowercase))

        self.assertEqual(masks[0].shape    ,			(1, 3, 17_64, 26_46))

        a__:     Tuple										=	[[1, 0], [0, 1]]
        with self.assertRaises(lowercase):
          a__:     List[Any]										=	processor.post_process_masks(lowercase    ,			np.array(lowercase)    ,			np.array(lowercase))







@require_vision
@require_tf
class    __snake_case       (     unittest.TestCase				):



      def   lowerCamelCase_   (						self)	->			List[Any]:





        '''simple docstring'''





        a__:     Optional[Any]										=	tempfile.mkdtemp()
        a__:     List[Any]										=	SamImageProcessor()
        a__:     Optional[int]										=	SamProcessor(lowercase)
        processor.save_pretrained(self.tmpdirname)



      def   lowerCamelCase_   (						self    ,			**lowercase)	->			int:





        '''simple docstring'''





        return AutoProcessor.from_pretrained(self.tmpdirname    ,			**lowercase).image_processor



      def   lowerCamelCase_   (						self)	->			Tuple:





        '''simple docstring'''





        shutil.rmtree(self.tmpdirname)



      def   lowerCamelCase_   (						self)	->			Tuple:





        '''simple docstring'''





        a__:     Optional[Any]										=	[np.random.randint(2_55    ,			size=(3, 30, 4_00)    ,			dtype=np.uinta)]

        a__:     Optional[int]										=	[Image.fromarray(np.moveaxis(lowercase    ,			0    ,			-1)) for x in image_inputs]

        return image_inputs



      def   lowerCamelCase_   (						self)	->			Any:





        '''simple docstring'''





        a__:     List[str]										=	SamProcessor(image_processor=self.get_image_processor())
        processor.save_pretrained(self.tmpdirname)

        a__:     Dict										=	self.get_image_processor(do_normalize=lowercase    ,			padding_value=1.0)

        a__:     Union[str, Any]										=	SamProcessor.from_pretrained(self.tmpdirname    ,			do_normalize=lowercase    ,			padding_value=1.0)

        self.assertEqual(processor.image_processor.to_json_string()    ,			image_processor_add_kwargs.to_json_string())
        self.assertIsInstance(processor.image_processor    ,			lowercase)



      def   lowerCamelCase_   (						self)	->			int:





        '''simple docstring'''





        a__:     Optional[Any]										=	self.get_image_processor()

        a__:     str										=	SamProcessor(image_processor=lowercase)

        a__:     int										=	self.prepare_image_inputs()

        a__:     int										=	image_processor(lowercase    ,			return_tensors='np')
        a__:     Dict										=	processor(images=lowercase    ,			return_tensors='np')

        input_feat_extract.pop('original_sizes')  # pop original_sizes as it is popped in the processor
        input_feat_extract.pop('reshaped_input_sizes')  # pop reshaped_input_sizes as it is popped in the processor

        for key in input_feat_extract.keys():
          self.assertAlmostEqual(input_feat_extract[key].sum()    ,			input_processor[key].sum()    ,			delta=1e-2)



      @require_tf
      def   lowerCamelCase_   (						self)	->			Any:





        '''simple docstring'''





        a__:     Tuple										=	self.get_image_processor()

        a__:     Any										=	SamProcessor(image_processor=lowercase)
        a__:     str										=	[tf.ones((1, 3, 5, 5))]

        a__:     List[Any]										=	[[17_64, 26_46]]

        a__:     List[Any]										=	[[6_83, 10_24]]
        a__:     List[Any]										=	processor.post_process_masks(lowercase    ,			lowercase    ,			lowercase    ,			return_tensors='tf')
        self.assertEqual(masks[0].shape    ,			(1, 3, 17_64, 26_46))

        a__:     Tuple										=	processor.post_process_masks(
            lowercase    ,			tf.convert_to_tensor(lowercase)    ,			tf.convert_to_tensor(lowercase)    ,			return_tensors='tf'    ,			)
        self.assertEqual(masks[0].shape    ,			(1, 3, 17_64, 26_46))

        # should also work with np
        a__:     Optional[Any]										=	[np.ones((1, 3, 5, 5))]
        a__:     int										=	processor.post_process_masks(
            lowercase    ,			np.array(lowercase)    ,			np.array(lowercase)    ,			return_tensors='tf')

        self.assertEqual(masks[0].shape    ,			(1, 3, 17_64, 26_46))

        a__:     List[str]										=	[[1, 0], [0, 1]]
        with self.assertRaises(tf.errors.InvalidArgumentError):
          a__:     Any										=	processor.post_process_masks(
              lowercase    ,			np.array(lowercase)    ,			np.array(lowercase)    ,			return_tensors='tf')







@require_vision
@require_torchvision
class    __snake_case       (     unittest.TestCase				):



      def   lowerCamelCase_   (						self)	->			Dict:





        '''simple docstring'''





        a__:     str										=	tempfile.mkdtemp()
        a__:     int										=	SamImageProcessor()
        a__:     Union[str, Any]										=	SamProcessor(lowercase)
        processor.save_pretrained(self.tmpdirname)



      def   lowerCamelCase_   (						self    ,			**lowercase)	->			Optional[int]:





        '''simple docstring'''





        return AutoProcessor.from_pretrained(self.tmpdirname    ,			**lowercase).image_processor



      def   lowerCamelCase_   (						self)	->			Tuple:





        '''simple docstring'''





        shutil.rmtree(self.tmpdirname)



      def   lowerCamelCase_   (						self)	->			Tuple:





        '''simple docstring'''





        a__:     Any										=	[np.random.randint(2_55    ,			size=(3, 30, 4_00)    ,			dtype=np.uinta)]

        a__:     Any										=	[Image.fromarray(np.moveaxis(lowercase    ,			0    ,			-1)) for x in image_inputs]

        return image_inputs



      @is_pt_tf_cross_test
      def   lowerCamelCase_   (						self)	->			int:





        '''simple docstring'''





        a__:     Optional[int]										=	self.get_image_processor()

        a__:     int										=	SamProcessor(image_processor=lowercase)
        a__:     int										=	np.random.randint(0    ,			2    ,			size=(1, 3, 5, 5)).astype(np.floataa)
        a__:     Dict										=	[tf.convert_to_tensor(lowercase)]
        a__:     Union[str, Any]										=	[torch.tensor(lowercase)]

        a__:     List[Any]										=	[[17_64, 26_46]]

        a__:     Optional[Any]										=	[[6_83, 10_24]]
        a__:     Tuple										=	processor.post_process_masks(
            lowercase    ,			lowercase    ,			lowercase    ,			return_tensors='tf')
        a__:     str										=	processor.post_process_masks(
            lowercase    ,			lowercase    ,			lowercase    ,			return_tensors='pt')

        self.assertTrue(np.all(tf_masks[0].numpy() == pt_masks[0].numpy()))



      @is_pt_tf_cross_test
      def   lowerCamelCase_   (						self)	->			str:





        '''simple docstring'''





        a__:     Tuple										=	self.get_image_processor()

        a__:     Dict										=	SamProcessor(image_processor=lowercase)

        a__:     Any										=	self.prepare_image_inputs()

        a__:     List[Any]										=	image_processor(lowercase    ,			return_tensors='pt')['pixel_values'].numpy()
        a__:     Tuple										=	processor(images=lowercase    ,			return_tensors='pt')['pixel_values'].numpy()

        a__:     Any										=	image_processor(lowercase    ,			return_tensors='tf')['pixel_values'].numpy()
        a__:     Any										=	processor(images=lowercase    ,			return_tensors='tf')['pixel_values'].numpy()

        self.assertTrue(np.allclose(lowercase    ,			lowercase))
        self.assertTrue(np.allclose(lowercase    ,			lowercase))
        self.assertTrue(np.allclose(lowercase    ,			lowercase))


