


from __future__ import annotations

import os
import tempfile
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import is_tensorflow_text_available, is_tf_available
from transformers.testing_utils import require_tensorflow_text, require_tf, slow

from ..test_modeling_tf_common import floats_tensor
from .test_framework_agnostic import GenerationIntegrationTestsMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import (
			    AutoTokenizer,
			    TFAutoModelForCausalLM,
			    TFAutoModelForSeqaSeqLM,
			    TFAutoModelForSpeechSeqaSeq,
			    TFAutoModelForVisionaSeq,
			    TFBartForConditionalGeneration,
			    TFLogitsProcessorList,
			    TFMinLengthLogitsProcessor,
			    tf_top_k_top_p_filtering,
			)

if is_tensorflow_text_available():
			import tensorflow_text as text


@require_tf
class 			_a				(     unittest.TestCase     ):

							def  lowerCamelCase_					(				self:      List[Any]				)     ->  Dict:





									"""simple docstring"""







									lowercase__				= tf.convert_to_tensor(
									    [
									        [
									            8.2220991,  # 3rd highest value; idx. 0
									            -0.5620044,
									            5.23229752,
									            4.0386393,
									            -6.8798378,
									            -0.54785802,
									            -3.2012153,
									            2.92777176,
									            1.88171953,
									            7.35341276,  # 5th highest value; idx. 9
									            8.43207833,  # 2nd highest value; idx. 10
									            -9.85711836,
									            -5.96209236,
									            -1.13039161,
									            -7.1115294,
									            -0.8369633,
									            -5.3186408,
									            7.06427407,
									            0.81369344,
									            -0.82023817,
									            -5.9179796,
									            0.58813443,
									            -6.99778438,
									            4.71551189,
									            -0.18771637,
									            7.44020759,  # 4th highest value; idx. 25
									            9.38450987,  # 1st highest value; idx. 26
									            2.12662941,
									            -9.32562038,
									            2.35652522,
									        ],  # cummulative prob of 5 highest values <= 0.6
									        [
									            0.58425518,
									            4.53139238,
									            -5.57510464,
									            -6.28030699,
									            -7.19529503,
									            -4.02122551,
									            1.39337037,
									            -6.06707057,
									            1.59480517,
									            -9.643119,
									            0.03907799,
									            0.67231762,
									            -8.88206726,
									            6.27115922,  # 4th highest value; idx. 13
									            2.28520723,
									            4.82767506,
									            4.30421368,
									            8.8275313,  # 2nd highest value; idx. 17
									            5.44029958,  # 5th highest value; idx. 18
									            -4.4735794,
									            7.38579536,  # 3rd highest value; idx. 20
									            -2.91051663,
									            2.61946077,
									            -2.5674762,
									            -9.48959302,
									            -4.02922645,
									            -1.35416918,
									            9.67702323,  # 1st highest value; idx. 27
									            -5.89478553,
									            1.85370467,
									        ],  # cummulative prob of 5 highest values <= 0.6
									    ]       ,		dtype=tf.floataa       ,		)

									lowercase__				= tf.convert_to_tensor(
									    [[0, 0], [0, 9], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 18], [1, 20], [1, 27]]       ,		dtype=tf.intaa       ,		)  # expected non filtered idx as noted above

									lowercase__				= tf.convert_to_tensor(
									    [8.222099, 7.3534126, 8.432078, 7.4402075, 9.38451, 6.271159, 8.827531, 5.4402995, 7.3857956, 9.677023]       ,		dtype=tf.floataa       ,		)  # expected non filtered values as noted above

									lowercase__				= tf_top_k_top_p_filtering(UpperCamelCase_       ,		top_k=10       ,		top_p=0.6       ,		min_tokens_to_keep=4				)

									lowercase__				= output[output != -float('''inf'''				)]
									lowercase__				= tf.cast(
									    tf.where(tf.not_equal(UpperCamelCase_       ,		tf.constant(-float('''inf'''				)       ,		dtype=tf.floataa				)				)				)       ,		dtype=tf.intaa       ,		)

									tf.debugging.assert_near(UpperCamelCase_       ,		UpperCamelCase_       ,		rtol=1E-1_2				)
									tf.debugging.assert_equal(UpperCamelCase_       ,		UpperCamelCase_				)


@require_tf
class 			_a				(     unittest.TestCase	,      UpperCamelCase__     ):
							# setting framework_dependent_parameters needs to be gated, just like its contents' imports
							if is_tf_available():
														_lowercase      :     List[Any]     =					{
														    '''AutoModelForCausalLM''': TFAutoModelForCausalLM,
														    '''AutoModelForSpeechSeq2Seq''': TFAutoModelForSpeechSeqaSeq,
														    '''AutoModelForSeq2SeqLM''': TFAutoModelForSeqaSeqLM,
														    '''AutoModelForVision2Seq''': TFAutoModelForVisionaSeq,
														    '''LogitsProcessorList''': TFLogitsProcessorList,
														    '''MinLengthLogitsProcessor''': TFMinLengthLogitsProcessor,
														    '''create_tensor_fn''': tf.convert_to_tensor,
														    '''floats_tensor''': floats_tensor,
														    '''return_tensors''': '''tf''',
														}

							@slow
							def  lowerCamelCase_					(				self:      Optional[int]				)     ->  Optional[int]:





									"""simple docstring"""







									lowercase__				= TFAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''				)
									lowercase__				= 2
									lowercase__				= 2


									class 			_a				(     tf.Module     ):

																def __init__(				self:      List[Any]       ,		UpperCamelCase_:      List[str]				)     ->  Optional[int]:





																		"""simple docstring"""







																		super(UpperCamelCase_       ,		self				).__init__()
																		lowercase__				= model


																@tf.function(
																    input_signature=(
																        tf.TensorSpec((None, input_length)       ,		tf.intaa       ,		name='''input_ids'''				),
																        tf.TensorSpec((None, input_length)       ,		tf.intaa       ,		name='''attention_mask'''				),
																    )       ,		jit_compile=UpperCamelCase_       ,		)
																def  lowerCamelCase_					(				self:      Optional[Any]       ,		UpperCamelCase_:      List[Any]       ,		UpperCamelCase_:      Dict				)     ->  Optional[Any]:





																		"""simple docstring"""







																		lowercase__				= self.model.generate(
																		    input_ids=UpperCamelCase_       ,		attention_mask=UpperCamelCase_       ,		max_new_tokens=UpperCamelCase_       ,		return_dict_in_generate=UpperCamelCase_       ,		)
																		return {"sequences": outputs["sequences"]}
									lowercase__				= [[2, 0], [102, 103]]
									lowercase__				= [[1, 0], [1, 1]]
									lowercase__				= DummyModel(model=UpperCamelCase_				)
									with tempfile.TemporaryDirectory() as tmp_dir:
											tf.saved_model.save(UpperCamelCase_       ,		UpperCamelCase_       ,		signatures={'''serving_default''': dummy_model.serving}				)
											lowercase__				= tf.saved_model.load(UpperCamelCase_				).signatures['''serving_default''']
											for batch_size in range(1       ,		len(UpperCamelCase_				) + 1				):
													lowercase__				= {
													    '''input_ids''': tf.constant(dummy_input_ids[:batch_size]				),
													    '''attention_mask''': tf.constant(dummy_attention_masks[:batch_size]				),
													}
													lowercase__				= serving_func(**UpperCamelCase_				)['''sequences''']
													lowercase__				= test_model.generate(**UpperCamelCase_       ,		max_new_tokens=UpperCamelCase_				)
													tf.debugging.assert_equal(UpperCamelCase_       ,		UpperCamelCase_				)

							@slow
							def  lowerCamelCase_					(				self:      Tuple				)     ->  Tuple:





									"""simple docstring"""







									lowercase__				= TFAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''				)
									lowercase__				= 1
									lowercase__				= 2


									class 			_a				(     tf.Module     ):

																def __init__(				self:      Tuple       ,		UpperCamelCase_:      Dict				)     ->  Tuple:





																		"""simple docstring"""







																		super(UpperCamelCase_       ,		self				).__init__()
																		lowercase__				= model


																@tf.function(
																    input_signature=(
																        tf.TensorSpec((batch_size, None)       ,		tf.intaa       ,		name='''input_ids'''				),
																        tf.TensorSpec((batch_size, None)       ,		tf.intaa       ,		name='''attention_mask'''				),
																    )       ,		jit_compile=UpperCamelCase_       ,		)
																def  lowerCamelCase_					(				self:      Any       ,		UpperCamelCase_:      Dict       ,		UpperCamelCase_:      List[str]				)     ->  Any:





																		"""simple docstring"""







																		lowercase__				= self.model.generate(
																		    input_ids=UpperCamelCase_       ,		attention_mask=UpperCamelCase_       ,		max_new_tokens=UpperCamelCase_       ,		return_dict_in_generate=UpperCamelCase_       ,		)
																		return {"sequences": outputs["sequences"]}
									lowercase__				= [[2], [102, 103]]
									lowercase__				= [[1], [1, 1]]
									lowercase__				= DummyModel(model=UpperCamelCase_				)
									with tempfile.TemporaryDirectory() as tmp_dir:
											tf.saved_model.save(UpperCamelCase_       ,		UpperCamelCase_       ,		signatures={'''serving_default''': dummy_model.serving}				)
											lowercase__				= tf.saved_model.load(UpperCamelCase_				).signatures['''serving_default''']
											for input_row in range(len(UpperCamelCase_				)				):
													lowercase__				= {
													    '''input_ids''': tf.constant([dummy_input_ids[input_row]]				),
													    '''attention_mask''': tf.constant([dummy_attention_masks[input_row]]				),
													}
													lowercase__				= serving_func(**UpperCamelCase_				)['''sequences''']
													lowercase__				= test_model.generate(**UpperCamelCase_       ,		max_new_tokens=UpperCamelCase_				)
													tf.debugging.assert_equal(UpperCamelCase_       ,		UpperCamelCase_				)

							@slow
							@require_tensorflow_text
							def  lowerCamelCase_					(				self:      Dict				)     ->  List[str]:





									"""simple docstring"""







									with tempfile.TemporaryDirectory() as tmp_dir:
											# file needed to load the TF tokenizer
											hf_hub_download(repo_id='''google/flan-t5-small'''       ,		filename='''spiece.model'''       ,		local_dir=UpperCamelCase_				)


											class 			_a				(     tf.keras.layers.Layer     ):

																		def __init__(				self:      str				)     ->  Tuple:





																				"""simple docstring"""







																				super().__init__()
																				lowercase__				= text.SentencepieceTokenizer(
																				    model=tf.io.gfile.GFile(os.path.join(UpperCamelCase_       ,		'''spiece.model'''				)       ,		'''rb'''				).read()				)
																				lowercase__				= TFAutoModelForSeqaSeqLM.from_pretrained('''hf-internal-testing/tiny-random-t5'''				)


																		def  lowerCamelCase_					(				self:      int       ,		UpperCamelCase_:      int       ,		*UpperCamelCase_:      List[Any]       ,		**UpperCamelCase_:      Dict				)     ->  Dict:





																				"""simple docstring"""







																				lowercase__				= self.tokenizer.tokenize(UpperCamelCase_				)
																				lowercase__	,						lowercase__				= text.pad_model_inputs(
																				    UpperCamelCase_       ,		max_seq_length=64       ,		pad_value=self.model.config.pad_token_id				)
																				lowercase__				= self.model.generate(input_ids=UpperCamelCase_       ,		attention_mask=UpperCamelCase_				)
																				return self.tokenizer.detokenize(UpperCamelCase_				)
											lowercase__				= CompleteSentenceTransformer()
											lowercase__				= tf.keras.layers.Input(shape=(1,)       ,		dtype=tf.string       ,		name='''inputs'''				)
											lowercase__				= complete_model(UpperCamelCase_				)
											lowercase__				= tf.keras.Model(UpperCamelCase_       ,		UpperCamelCase_				)
											keras_model.save(UpperCamelCase_				)

							def  lowerCamelCase_					(				self:      Dict				)     ->  Tuple:





									"""simple docstring"""







									lowercase__				= {
									    '''do_sample''': True,
									    '''num_beams''': 1,
									    '''top_p''': 0.7,
									    '''top_k''': 10,
									    '''temperature''': 0.7,
									}
									lowercase__				= 14

									lowercase__				= AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''				)
									lowercase__				= '''Hello, my dog is cute and'''
									lowercase__				= tokenizer(UpperCamelCase_       ,		return_tensors='''tf'''				)
									lowercase__				= TFAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''				)

									lowercase__				= 638
									# forces the generation to happen on CPU, to avoid GPU-related quirks
									with tf.device(''':/CPU:0'''				):
											tf.random.set_seed(0				)
											lowercase__				= model.generate(**UpperCamelCase_       ,		eos_token_id=UpperCamelCase_       ,		**UpperCamelCase_				)
									self.assertTrue(expectation == len(generated_tokens[0]				)				)

									lowercase__				= [638, 198]
									with tf.device(''':/CPU:0'''				):
											tf.random.set_seed(0				)
											lowercase__				= model.generate(**UpperCamelCase_       ,		eos_token_id=UpperCamelCase_       ,		**UpperCamelCase_				)
									self.assertTrue(expectation == len(generated_tokens[0]				)				)


							def  lowerCamelCase_					(				self:      Optional[int]				)     ->  Dict:





									"""simple docstring"""







									lowercase__				= AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bart'''				)
									lowercase__				= '''Hugging Face is a technology company based in New York and Paris.'''
									lowercase__				= bart_tokenizer(UpperCamelCase_       ,		return_tensors='''tf'''				).input_ids
									lowercase__				= TFBartForConditionalGeneration.from_pretrained('''hf-internal-testing/tiny-random-bart'''				)
									lowercase__				= bart_model.generate(UpperCamelCase_				).numpy()


									class 			_a				(     UpperCamelCase__     ):

																def  lowerCamelCase_					(				self:      Optional[int]       ,		UpperCamelCase_:      Optional[int]       ,		UpperCamelCase_:      List[Any]=None       ,		**UpperCamelCase_:      Optional[int]				)     ->  List[Any]:





																		"""simple docstring"""







																		return super().call(UpperCamelCase_       ,		**UpperCamelCase_				)
									lowercase__				= FakeBart.from_pretrained('''hf-internal-testing/tiny-random-bart'''				)
									lowercase__				= bart_model.generate(UpperCamelCase_       ,		foo='''bar'''				).numpy()
									self.assertTrue(np.array_equal(UpperCamelCase_       ,		UpperCamelCase_				)				)


									class 			_a				(     bart_model.model.encoder.__class__     ):

																def  lowerCamelCase_					(				self:      List[Any]       ,		UpperCamelCase_:      Tuple       ,		**UpperCamelCase_:      int				)     ->  List[Any]:





																		"""simple docstring"""







																		return super().call(UpperCamelCase_       ,		**UpperCamelCase_				)
									lowercase__				= FakeEncoder(bart_model.config       ,		bart_model.model.shared				)
									lowercase__				= fake_encoder

									# Normal generation still works (the output will be different because the encoder weights are different)
									lowercase__				= bart_model.generate(UpperCamelCase_				).numpy()
									with self.assertRaises(UpperCamelCase_				):
											# FakeEncoder.call() accepts **kwargs -> no filtering -> value error due to unexpected input "foo"
											bart_model.generate(UpperCamelCase_       ,		foo='''bar'''				)







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


_lowercase    	=		{
    "configuration_layoutlmv2": ["LAYOUTLMV2_PRETRAINED_CONFIG_ARCHIVE_MAP", "LayoutLMv2Config"],
    "processing_layoutlmv2": ["LayoutLMv2Processor"],
    "tokenization_layoutlmv2": ["LayoutLMv2Tokenizer"],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _lowercase    	=		["LayoutLMv2TokenizerFast"]

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _lowercase    	=		["LayoutLMv2FeatureExtractor"]
   _lowercase    	=		["LayoutLMv2ImageProcessor"]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _lowercase    	=		[
       "LAYOUTLMV2_PRETRAINED_MODEL_ARCHIVE_LIST",
       "LayoutLMv2ForQuestionAnswering",
       "LayoutLMv2ForSequenceClassification",
       "LayoutLMv2ForTokenClassification",
       "LayoutLMv2Layer",
       "LayoutLMv2Model",
       "LayoutLMv2PreTrainedModel",
   ]

if TYPE_CHECKING:
   from .configuration_layoutlmva import LAYOUTLMV2_PRETRAINED_CONFIG_ARCHIVE_MAP, LayoutLMvaConfig
   from .processing_layoutlmva import LayoutLMvaProcessor
   from .tokenization_layoutlmva import LayoutLMvaTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_layoutlmva_fast import LayoutLMvaTokenizerFast

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .feature_extraction_layoutlmva import LayoutLMvaFeatureExtractor, LayoutLMvaImageProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_layoutlmva import (
          LAYOUTLMV2_PRETRAINED_MODEL_ARCHIVE_LIST,
          LayoutLMvaForQuestionAnswering,
          LayoutLMvaForSequenceClassification,
          LayoutLMvaForTokenClassification,
          LayoutLMvaLayer,
          LayoutLMvaModel,
          LayoutLMvaPreTrainedModel,
      )
else:
   import sys

   _lowercase    	=		_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
