


import json
import os
import shutil
import tempfile
import unittest

import numpy as np

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES, BertTokenizer
from transformers.testing_utils import require_tokenizers, require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
		from PIL import Image

		from transformers import VisionTextDualEncoderProcessor, ViTImageProcessor

@require_tokenizers
@require_vision
class 						__a (							unittest.TestCase     ):
		def       UpperCAmelCase__							(     self		)					->			Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase											=     tempfile.mkdtemp()

									# fmt: off
									_UpperCAmelCase											=     ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing', ',', 'low', 'lowest']
									# fmt: on
									_UpperCAmelCase											=     os.path.join(self.tmpdirname     ,       VOCAB_FILES_NAMES['vocab_file']		)
									with open(self.vocab_file     ,       'w'     ,       encoding='utf-8'		) as vocab_writer:
																vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]		)		)

									_UpperCAmelCase											=     {
									    'do_resize': True,
									    'size': {'height': 18, 'width': 18},
									    'do_normalize': True,
									    'image_mean': [0.5, 0.5, 0.5],
									    'image_std': [0.5, 0.5, 0.5],
									}
									_UpperCAmelCase											=     os.path.join(self.tmpdirname     ,       _SCREAMING_SNAKE_CASE		)
									with open(self.image_processor_file     ,       'w'     ,       encoding='utf-8'		) as fp:
																json.dump(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)
		def       UpperCAmelCase__							(     self     ,       **_SCREAMING_SNAKE_CASE		)					->			Optional[int]:




									"""simple docstring"""





									return BertTokenizer.from_pretrained(self.tmpdirname     ,       **_SCREAMING_SNAKE_CASE		)
		def       UpperCAmelCase__							(     self     ,       **_SCREAMING_SNAKE_CASE		)					->			Union[str, Any]:




									"""simple docstring"""





									return ViTImageProcessor.from_pretrained(self.tmpdirname     ,       **_SCREAMING_SNAKE_CASE		)
		def       UpperCAmelCase__							(     self		)					->			int:




									"""simple docstring"""





									shutil.rmtree(self.tmpdirname		)
		def       UpperCAmelCase__							(     self		)					->			Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase											=     [np.random.randint(255     ,       size=(3, 30, 400)     ,       dtype=np.uinta		)]

									_UpperCAmelCase											=     [Image.fromarray(np.moveaxis(_SCREAMING_SNAKE_CASE     ,       0     ,       -1		)		) for x in image_inputs]

									return image_inputs
		def       UpperCAmelCase__							(     self		)					->			Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase											=     self.get_tokenizer()
									_UpperCAmelCase											=     self.get_image_processor()

									_UpperCAmelCase											=     VisionTextDualEncoderProcessor(tokenizer=_SCREAMING_SNAKE_CASE     ,       image_processor=_SCREAMING_SNAKE_CASE		)

									processor.save_pretrained(self.tmpdirname		)
									_UpperCAmelCase											=     VisionTextDualEncoderProcessor.from_pretrained(self.tmpdirname		)

									self.assertEqual(processor.tokenizer.get_vocab()     ,       tokenizer.get_vocab()		)
									self.assertIsInstance(processor.tokenizer     ,       (BertTokenizer, BertTokenizerFast)		)

									self.assertEqual(processor.image_processor.to_json_string()     ,       image_processor.to_json_string()		)
									self.assertIsInstance(processor.image_processor     ,       _SCREAMING_SNAKE_CASE		)
		def       UpperCAmelCase__							(     self		)					->			Any:




									"""simple docstring"""





									_UpperCAmelCase											=     VisionTextDualEncoderProcessor(
									    tokenizer=self.get_tokenizer()     ,       image_processor=self.get_image_processor()		)
									processor.save_pretrained(self.tmpdirname		)

									_UpperCAmelCase											=     self.get_tokenizer(bos_token='(BOS)'     ,       eos_token='(EOS)'		)
									_UpperCAmelCase											=     self.get_image_processor(do_normalize=_SCREAMING_SNAKE_CASE     ,       padding_value=1.0		)

									_UpperCAmelCase											=     VisionTextDualEncoderProcessor.from_pretrained(
									    self.tmpdirname     ,       bos_token='(BOS)'     ,       eos_token='(EOS)'     ,       do_normalize=_SCREAMING_SNAKE_CASE     ,       padding_value=1.0		)

									self.assertEqual(processor.tokenizer.get_vocab()     ,       tokenizer_add_kwargs.get_vocab()		)
									self.assertIsInstance(processor.tokenizer     ,       (BertTokenizer, BertTokenizerFast)		)

									self.assertEqual(processor.image_processor.to_json_string()     ,       image_processor_add_kwargs.to_json_string()		)
									self.assertIsInstance(processor.image_processor     ,       _SCREAMING_SNAKE_CASE		)
		def       UpperCAmelCase__							(     self		)					->			List[Any]:




									"""simple docstring"""





									_UpperCAmelCase											=     self.get_image_processor()
									_UpperCAmelCase											=     self.get_tokenizer()

									_UpperCAmelCase											=     VisionTextDualEncoderProcessor(tokenizer=_SCREAMING_SNAKE_CASE     ,       image_processor=_SCREAMING_SNAKE_CASE		)

									_UpperCAmelCase											=     self.prepare_image_inputs()

									_UpperCAmelCase											=     image_processor(_SCREAMING_SNAKE_CASE     ,       return_tensors='np'		)
									_UpperCAmelCase											=     processor(images=_SCREAMING_SNAKE_CASE     ,       return_tensors='np'		)

									for key in input_feat_extract.keys():
																self.assertAlmostEqual(input_feat_extract[key].sum()     ,       input_processor[key].sum()     ,       delta=1e-2		)
		def       UpperCAmelCase__							(     self		)					->			Tuple:




									"""simple docstring"""





									_UpperCAmelCase											=     self.get_image_processor()
									_UpperCAmelCase											=     self.get_tokenizer()

									_UpperCAmelCase											=     VisionTextDualEncoderProcessor(tokenizer=_SCREAMING_SNAKE_CASE     ,       image_processor=_SCREAMING_SNAKE_CASE		)

									_UpperCAmelCase											=     'lower newer'

									_UpperCAmelCase											=     processor(text=_SCREAMING_SNAKE_CASE		)

									_UpperCAmelCase											=     tokenizer(_SCREAMING_SNAKE_CASE		)

									for key in encoded_tok.keys():
																self.assertListEqual(encoded_tok[key]     ,       encoded_processor[key]		)
		def       UpperCAmelCase__							(     self		)					->			Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase											=     self.get_image_processor()
									_UpperCAmelCase											=     self.get_tokenizer()

									_UpperCAmelCase											=     VisionTextDualEncoderProcessor(tokenizer=_SCREAMING_SNAKE_CASE     ,       image_processor=_SCREAMING_SNAKE_CASE		)

									_UpperCAmelCase											=     'lower newer'
									_UpperCAmelCase											=     self.prepare_image_inputs()

									_UpperCAmelCase											=     processor(text=_SCREAMING_SNAKE_CASE     ,       images=_SCREAMING_SNAKE_CASE		)

									self.assertListEqual(list(inputs.keys()		)     ,       ['input_ids', 'token_type_ids', 'attention_mask', 'pixel_values']		)

									# test if it raises when no input is passed
									with self.assertRaises(_SCREAMING_SNAKE_CASE		):
																processor()
		def       UpperCAmelCase__							(     self		)					->			List[str]:




									"""simple docstring"""





									_UpperCAmelCase											=     self.get_image_processor()
									_UpperCAmelCase											=     self.get_tokenizer()

									_UpperCAmelCase											=     VisionTextDualEncoderProcessor(tokenizer=_SCREAMING_SNAKE_CASE     ,       image_processor=_SCREAMING_SNAKE_CASE		)

									_UpperCAmelCase											=     [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

									_UpperCAmelCase											=     processor.batch_decode(_SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase											=     tokenizer.batch_decode(_SCREAMING_SNAKE_CASE		)

									self.assertListEqual(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)






		def       UpperCAmelCase__							(     self		)					->			Any:




									"""simple docstring"""





									_UpperCAmelCase											=     self.get_image_processor()
									_UpperCAmelCase											=     self.get_tokenizer()

									_UpperCAmelCase											=     VisionTextDualEncoderProcessor(tokenizer=_SCREAMING_SNAKE_CASE     ,       image_processor=_SCREAMING_SNAKE_CASE		)

									_UpperCAmelCase											=     'lower newer'
									_UpperCAmelCase											=     self.prepare_image_inputs()

									_UpperCAmelCase											=     processor(text=_SCREAMING_SNAKE_CASE     ,       images=_SCREAMING_SNAKE_CASE		)

									self.assertListEqual(list(inputs.keys()		)     ,       processor.model_input_names		)




import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoFeatureExtractor, WavaVecaFeatureExtractor
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


lowerCAmelCase__    :Optional[Any]       =				get_tests_dir('''fixtures''')

class 						__a (							unittest.TestCase     ):
		def       UpperCAmelCase__							(     self		)					->			Any:




									"""simple docstring"""





									_UpperCAmelCase											=     mock.Mock()
									_UpperCAmelCase											=     500
									_UpperCAmelCase											=     {}
									_UpperCAmelCase											=     HTTPError
									_UpperCAmelCase											=     {}

									# Download this model to make sure it's in the cache.
									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained('hf-internal-testing/tiny-random-wav2vec2'		)
									# Under the mock environment we get a 500 error when trying to reach the model.
									with mock.patch('requests.Session.request'     ,       return_value=_SCREAMING_SNAKE_CASE		) as mock_head:
																_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained('hf-internal-testing/tiny-random-wav2vec2'		)
																# This check we did call the fake head request
																mock_head.assert_called()






		def       UpperCAmelCase__							(     self		)					->			Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained(
									    'https://huggingface.co/hf-internal-testing/tiny-random-wav2vec2/resolve/main/preprocessor_config.json'		)



@is_staging_test
class 						__a (							unittest.TestCase     ):
		@classmethod
		def       UpperCAmelCase__							(     cls		)					->			Tuple:




									"""simple docstring"""





									_UpperCAmelCase											=     TOKEN
									HfFolder.save_token(_SCREAMING_SNAKE_CASE		)
		@classmethod
		def       UpperCAmelCase__							(     cls		)					->			Tuple:




									"""simple docstring"""





									try:
																delete_repo(token=cls._token     ,       repo_id='test-feature-extractor'		)
									except HTTPError:
																pass

									try:
																delete_repo(token=cls._token     ,       repo_id='valid_org/test-feature-extractor-org'		)
									except HTTPError:
																pass

									try:
																delete_repo(token=cls._token     ,       repo_id='test-dynamic-feature-extractor'		)
									except HTTPError:
																pass
		def       UpperCAmelCase__							(     self		)					->			Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained(_SCREAMING_SNAKE_CASE		)
									feature_extractor.push_to_hub('test-feature-extractor'     ,       use_auth_token=self._token		)

									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained(f'''{USER}/test-feature-extractor'''		)
									for k, v in feature_extractor.__dict__.items():
																self.assertEqual(_SCREAMING_SNAKE_CASE     ,       getattr(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)		)

									# Reset repo
									delete_repo(token=self._token     ,       repo_id='test-feature-extractor'		)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
																feature_extractor.save_pretrained(
																    _SCREAMING_SNAKE_CASE     ,       repo_id='test-feature-extractor'     ,       push_to_hub=_SCREAMING_SNAKE_CASE     ,       use_auth_token=self._token		)

									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained(f'''{USER}/test-feature-extractor'''		)
									for k, v in feature_extractor.__dict__.items():
																self.assertEqual(_SCREAMING_SNAKE_CASE     ,       getattr(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)		)
		def       UpperCAmelCase__							(     self		)					->			Optional[Any]:




									"""simple docstring"""





									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained(_SCREAMING_SNAKE_CASE		)
									feature_extractor.push_to_hub('valid_org/test-feature-extractor'     ,       use_auth_token=self._token		)

									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained('valid_org/test-feature-extractor'		)
									for k, v in feature_extractor.__dict__.items():
																self.assertEqual(_SCREAMING_SNAKE_CASE     ,       getattr(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)		)

									# Reset repo
									delete_repo(token=self._token     ,       repo_id='valid_org/test-feature-extractor'		)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
																feature_extractor.save_pretrained(
																    _SCREAMING_SNAKE_CASE     ,       repo_id='valid_org/test-feature-extractor-org'     ,       push_to_hub=_SCREAMING_SNAKE_CASE     ,       use_auth_token=self._token		)

									_UpperCAmelCase											=     WavaVecaFeatureExtractor.from_pretrained('valid_org/test-feature-extractor-org'		)
									for k, v in feature_extractor.__dict__.items():
																self.assertEqual(_SCREAMING_SNAKE_CASE     ,       getattr(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)		)






		def       UpperCAmelCase__							(     self		)					->			Tuple:




									"""simple docstring"""





									CustomFeatureExtractor.register_for_auto_class()
									_UpperCAmelCase											=     CustomFeatureExtractor.from_pretrained(_SCREAMING_SNAKE_CASE		)

									feature_extractor.push_to_hub('test-dynamic-feature-extractor'     ,       use_auth_token=self._token		)

									# This has added the proper auto_map field to the config
									self.assertDictEqual(
									    feature_extractor.auto_map     ,       {'AutoFeatureExtractor': 'custom_feature_extraction.CustomFeatureExtractor'}     ,       )

									_UpperCAmelCase											=     AutoFeatureExtractor.from_pretrained(
									    f'''{USER}/test-dynamic-feature-extractor'''     ,       trust_remote_code=_SCREAMING_SNAKE_CASE		)
									# Can't make an isinstance check because the new_feature_extractor is from the CustomFeatureExtractor class of a dynamic module
									self.assertEqual(new_feature_extractor.__class__.__name__     ,       'CustomFeatureExtractor'		)
