




"""simple docstring"""






import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType






class lowerCAmelCase__	(     lowercase						):






			'''simple docstring'''



			lowerCamelCase__      =			["""image_processor""", """tokenizer"""]
			lowerCamelCase__      =			"""LayoutLMv2ImageProcessor"""
			lowerCamelCase__      =			("""LayoutXLMTokenizer""", """LayoutXLMTokenizerFast""")
			def __init__(  self  ,      lowercase=None  ,      lowercase=None  ,      **lowercase  ):
							if "feature_extractor" in kwargs:
											warnings.warn(
											    'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
											    ' instead.'  ,      lowercase  ,      )
											_lowerCamelCase     :			Union[str, Any]    =    kwargs.pop('feature_extractor'  )

							_lowerCamelCase     :			Optional[int]    =    image_processor if image_processor is not None else feature_extractor
							if image_processor is None:
											raise ValueError('You need to specify an `image_processor`.'  )
							if tokenizer is None:
											raise ValueError('You need to specify a `tokenizer`.'  )

							super().__init__(lowercase  ,      lowercase  )
			def __call__(  self  ,      lowercase  ,      lowercase = None  ,      lowercase = None  ,      lowercase = None  ,      lowercase = None  ,      lowercase = True  ,      lowercase = False  ,      lowercase = None  ,      lowercase = None  ,      lowercase = 0  ,      lowercase = None  ,      lowercase = None  ,      lowercase = None  ,      lowercase = False  ,      lowercase = False  ,      lowercase = False  ,      lowercase = False  ,      lowercase = True  ,      lowercase = None  ,      **lowercase  ,      ):
							# verify input
							if self.image_processor.apply_ocr and (boxes is not None):
											raise ValueError(
											    'You cannot provide bounding boxes '
											    'if you initialized the image processor with apply_ocr set to True.'  )

							if self.image_processor.apply_ocr and (word_labels is not None):
											raise ValueError(
											    'You cannot provide word labels if you initialized the image processor with apply_ocr set to True.'  )

							if return_overflowing_tokens is True and return_offsets_mapping is False:
											raise ValueError('You cannot return overflowing tokens without returning the offsets mapping.'  )

							# first, apply the image processor
							_lowerCamelCase     :			int    =    self.image_processor(images=lowercase  ,      return_tensors=lowercase  )

							# second, apply the tokenizer
							if text is not None and self.image_processor.apply_ocr and text_pair is None:
											if isinstance(lowercase  ,      lowercase  ):
															_lowerCamelCase     :			Tuple    =    [text]  # add batch dimension (as the image processor always adds a batch dimension)
											_lowerCamelCase     :			List[Any]    =    features['words']

							_lowerCamelCase     :			Any    =    self.tokenizer(
							    text=text if text is not None else features['words']  ,      text_pair=text_pair if text_pair is not None else None  ,      boxes=boxes if boxes is not None else features['boxes']  ,      word_labels=lowercase  ,      add_special_tokens=lowercase  ,      padding=lowercase  ,      truncation=lowercase  ,      max_length=lowercase  ,      stride=lowercase  ,      pad_to_multiple_of=lowercase  ,      return_token_type_ids=lowercase  ,      return_attention_mask=lowercase  ,      return_overflowing_tokens=lowercase  ,      return_special_tokens_mask=lowercase  ,      return_offsets_mapping=lowercase  ,      return_length=lowercase  ,      verbose=lowercase  ,      return_tensors=lowercase  ,      **lowercase  ,      )

							# add pixel values
							_lowerCamelCase     :			Dict    =    features.pop('pixel_values'  )
							if return_overflowing_tokens is True:
											_lowerCamelCase     :			List[Any]    =    self.get_overflowing_images(lowercase  ,      encoded_inputs['overflow_to_sample_mapping']  )
							_lowerCamelCase     :			int    =    images

							return encoded_inputs
			def 					A_    (  self  ,      lowercase  ,      lowercase  ):
							# in case there's an overflow, ensure each `input_ids` sample is mapped to its corresponding image
							_lowerCamelCase     :			Dict    =    []
							for sample_idx in overflow_to_sample_mapping:
											images_with_overflow.append(images[sample_idx]  )

							if len(lowercase  ) != len(lowercase  ):
											raise ValueError(
											    'Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got'
											    F''' {len(lowercase  )} and {len(lowercase  )}'''  )

							return images_with_overflow
			def 					A_    (  self  ,      *lowercase  ,      **lowercase  ):
							return self.tokenizer.batch_decode(*lowercase  ,      **lowercase  )
			def 					A_    (  self  ,      *lowercase  ,      **lowercase  ):
							return self.tokenizer.decode(*lowercase  ,      **lowercase  )
			@property
			def 					A_    (  self  ):
							return ["input_ids", "bbox", "attention_mask", "image"]
			@property
			def 					A_    (  self  ):
							warnings.warn(
							    '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'  ,      lowercase  ,      )
							return self.image_processor_class






			@property
			def 					A_    (  self  ):
							warnings.warn(
							    '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'  ,      lowercase  ,      )
							return self.image_processor





"""simple docstring"""






from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
			from transformers import (
			    TFAutoModel,
			    TFAutoModelForCausalLM,
			    TFAutoModelForMaskedLM,
			    TFAutoModelForPreTraining,
			    TFAutoModelForQuestionAnswering,
			    TFAutoModelForSeqaSeqLM,
			    TFAutoModelForSequenceClassification,
			    TFAutoModelForTableQuestionAnswering,
			    TFAutoModelForTokenClassification,
			    TFAutoModelWithLMHead,
			    TFBertForMaskedLM,
			    TFBertForPreTraining,
			    TFBertForQuestionAnswering,
			    TFBertForSequenceClassification,
			    TFBertModel,
			    TFFunnelBaseModel,
			    TFFunnelModel,
			    TFGPTaLMHeadModel,
			    TFRobertaForMaskedLM,
			    TFTaForConditionalGeneration,
			    TFTapasForQuestionAnswering,
			)
			from transformers.models.auto.modeling_tf_auto import (
			    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
			    TF_MODEL_FOR_MASKED_LM_MAPPING,
			    TF_MODEL_FOR_PRETRAINING_MAPPING,
			    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
			    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
			    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
			    TF_MODEL_MAPPING,
			)
			from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST






class lowerCAmelCase__	(     lowercase						):






			'''simple docstring'''



			lowerCamelCase__      =			"""new-model"""


if is_tf_available():






			class lowerCAmelCase__	(     lowercase						):






						'''simple docstring'''



						lowerCamelCase__      =			NewModelConfig






@require_tf
class lowerCAmelCase__	(     unittest.TestCase						):






			'''simple docstring'''



			@slow
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[str]    =    'bert-base-cased'
							_lowerCamelCase     :			Union[str, Any]    =    AutoConfig.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							_lowerCamelCase     :			Union[str, Any]    =    TFAutoModel.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[str]    =    'bert-base-cased'
							_lowerCamelCase     :			int    =    AutoConfig.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							_lowerCamelCase     :			int    =    TFAutoModelForPreTraining.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			Dict    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			int    =    TFAutoModelForCausalLM.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			str    =    TFAutoModelForCausalLM.from_pretrained(lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			List[Any]    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			str    =    TFAutoModelWithLMHead.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			Tuple    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Optional[int]    =    TFAutoModelForMaskedLM.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			Tuple    =    TFAutoModelForMaskedLM.from_pretrained(lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			Optional[int]    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Tuple    =    TFAutoModelForSeqaSeqLM.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			Tuple    =    TFAutoModelForSeqaSeqLM.from_pretrained(lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							for model_name in ["bert-base-uncased"]:
											_lowerCamelCase     :			str    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Union[str, Any]    =    TFAutoModelForSequenceClassification.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							for model_name in ["bert-base-uncased"]:
											_lowerCamelCase     :			Optional[Any]    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			List[str]    =    TFAutoModelForQuestionAnswering.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			@require_tensorflow_probability
			def 					A_    (  self  ):
							for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
											_lowerCamelCase     :			Dict    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Dict    =    TFAutoModelForTableQuestionAnswering.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			List[Any]    =    TFAutoModelForTableQuestionAnswering.from_pretrained(
											    lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			def 					A_    (  self  ):
							_lowerCamelCase     :			int    =    TFAutoModelWithLMHead.from_pretrained(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
							self.assertEqual(model.num_parameters()  ,      14410  )
							self.assertEqual(model.num_parameters(only_trainable=lowercase  )  ,      14410  )
			def 					A_    (  self  ):
							_lowerCamelCase     :			Any    =    TFAutoModelWithLMHead.from_pretrained(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
							self.assertEqual(model.num_parameters()  ,      14410  )
							self.assertEqual(model.num_parameters(only_trainable=lowercase  )  ,      14410  )
			def 					A_    (  self  ):
							# For the auto model mapping, FunnelConfig has two models: FunnelModel and FunnelBaseModel
							_lowerCamelCase     :			List[str]    =    TFAutoModel.from_pretrained('sgugger/funnel-random-tiny'  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							_lowerCamelCase     :			Optional[int]    =    copy.deepcopy(model.config  )
							_lowerCamelCase     :			Dict    =    ['FunnelBaseModel']
							_lowerCamelCase     :			List[Any]    =    TFAutoModel.from_config(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							with tempfile.TemporaryDirectory() as tmp_dir:
											model.save_pretrained(lowercase  )
											_lowerCamelCase     :			Tuple    =    TFAutoModel.from_pretrained(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			def 					A_    (  self  ):
							try:
											AutoConfig.register('new-model'  ,      lowercase  )

											_lowerCamelCase     :			Tuple    =    [
											    TFAutoModel,
											    TFAutoModelForCausalLM,
											    TFAutoModelForMaskedLM,
											    TFAutoModelForPreTraining,
											    TFAutoModelForQuestionAnswering,
											    TFAutoModelForSequenceClassification,
											    TFAutoModelForTokenClassification,
											]

											for auto_class in auto_classes:
															with self.subTest(auto_class.__name__  ):
																			# Wrong config class will raise an error
																			with self.assertRaises(lowercase  ):
																							auto_class.register(lowercase  ,      lowercase  )
																			auto_class.register(lowercase  ,      lowercase  )
																			# Trying to register something existing in the Transformers library will raise an error
																			with self.assertRaises(lowercase  ):
																							auto_class.register(lowercase  ,      lowercase  )

																			# Now that the config is registered, it can be used as any other config with the auto-API
																			_lowerCamelCase     :			Optional[Any]    =    BertModelTester(self  ).get_config()
																			_lowerCamelCase     :			Dict    =    NewModelConfig(**tiny_config.to_dict()  )
																			_lowerCamelCase     :			int    =    auto_class.from_config(lowercase  )
																			self.assertIsInstance(lowercase  ,      lowercase  )

																			with tempfile.TemporaryDirectory() as tmp_dir:
																							model.save_pretrained(lowercase  )
																							_lowerCamelCase     :			List[Any]    =    auto_class.from_pretrained(lowercase  )
																							self.assertIsInstance(lowercase  ,      lowercase  )

							finally:
											if "new-model" in CONFIG_MAPPING._extra_content:
															del CONFIG_MAPPING._extra_content["new-model"]
											for mapping in (
											    TF_MODEL_MAPPING,
											    TF_MODEL_FOR_PRETRAINING_MAPPING,
											    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
											    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
											    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
											    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
											    TF_MODEL_FOR_MASKED_LM_MAPPING,
											):
															if NewModelConfig in mapping._extra_content:
																			del mapping._extra_content[NewModelConfig]
			def 					A_    (  self  ):
							with self.assertRaisesRegex(
							    lowercase  ,      'bert-base is not a local folder and is not a valid model identifier'  ):
											_lowerCamelCase     :			Union[str, Any]    =    TFAutoModel.from_pretrained('bert-base'  )
			def 					A_    (  self  ):
							with self.assertRaisesRegex(
							    lowercase  ,      r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'  ):
											_lowerCamelCase     :			str    =    TFAutoModel.from_pretrained(lowercase  ,      revision='aaaaaa'  )
			def 					A_    (  self  ):
							with self.assertRaisesRegex(
							    lowercase  ,      'hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'  ,      ):
											_lowerCamelCase     :			Tuple    =    TFAutoModel.from_pretrained('hf-internal-testing/config-no-model'  )
			def 					A_    (  self  ):
							with self.assertRaisesRegex(lowercase  ,      'Use `from_pt=True` to load this model'  ):
											_lowerCamelCase     :			Tuple    =    TFAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only'  )






			def 					A_    (  self  ):
							# Make sure we have cached the model.
							_lowerCamelCase     :			Optional[int]    =    TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'  )
							with RequestCounter() as counter:
											_lowerCamelCase     :			Optional[int]    =    TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'  )
											self.assertEqual(counter.get_request_count  ,      0  )
											self.assertEqual(counter.head_request_count  ,      1  )
											self.assertEqual(counter.other_request_count  ,      0  )

							# With a sharded checkpoint
							_lowerCamelCase     :			int    =    TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'  )
							with RequestCounter() as counter:
											_lowerCamelCase     :			List[Any]    =    TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'  )
											self.assertEqual(counter.get_request_count  ,      0  )
											self.assertEqual(counter.head_request_count  ,      1  )
											self.assertEqual(counter.other_request_count  ,      0  )