





from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import numpy
				import tensorflow as tf

				from transformers import (
				    TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    BertConfig,
				    DPRConfig,
				    TFDPRContextEncoder,
				    TFDPRQuestionEncoder,
				    TFDPRReader,
				)





class 						_A:


				"""simple docstring"""


				def __init__(			self     ,					_A     ,					_A=13     ,					_A=7     ,					_A=True     ,					_A=True     ,					_A=True     ,					_A=True     ,					_A=99     ,					_A=32     ,					_A=2     ,					_A=4     ,					_A=37     ,					_A="gelu"     ,					_A=0.1     ,					_A=0.1     ,					_A=512     ,					_A=16     ,					_A=2     ,					_A=0.0_2     ,					_A=3     ,					_A=4     ,					_A=None     ,					_A=0     ,					):
									__A	:						Dict       =				parent
									__A	:						List[str]       =				batch_size
									__A	:						int       =				seq_length
									__A	:						List[str]       =				is_training
									__A	:						List[Any]       =				use_input_mask
									__A	:						Dict       =				use_token_type_ids
									__A	:						List[Any]       =				use_labels
									__A	:						List[Any]       =				vocab_size
									__A	:						Tuple       =				hidden_size
									__A	:						Union[str, Any]       =				num_hidden_layers
									__A	:						int       =				num_attention_heads
									__A	:						Tuple       =				intermediate_size
									__A	:						str       =				hidden_act
									__A	:						Optional[int]       =				hidden_dropout_prob
									__A	:						Optional[int]       =				attention_probs_dropout_prob
									__A	:						int       =				max_position_embeddings
									__A	:						Dict       =				type_vocab_size
									__A	:						int       =				type_sequence_label_size
									__A	:						Optional[int]       =				initializer_range
									__A	:						Any       =				num_labels
									__A	:						Any       =				num_choices
									__A	:						int       =				scope
									__A	:						Tuple       =				projection_dim



				def 					UpperCAmelCase_		(			self     ):
									__A	:						Any       =				ids_tensor([self.batch_size, self.seq_length]     ,					self.vocab_size     )

									__A	:						Any       =				None
									if self.use_input_mask:
														# follow test_modeling_tf_ctrl.py
														__A	:						List[Any]       =				random_attention_mask([self.batch_size, self.seq_length]     )

									__A	:						int       =				None
									if self.use_token_type_ids:
														__A	:						Union[str, Any]       =				ids_tensor([self.batch_size, self.seq_length]     ,					self.type_vocab_size     )

									__A	:						str       =				None
									__A	:						Tuple       =				None
									__A	:						Optional[Any]       =				None
									if self.use_labels:
														__A	:						Dict       =				ids_tensor([self.batch_size]     ,					self.type_sequence_label_size     )
														__A	:						List[Any]       =				ids_tensor([self.batch_size, self.seq_length]     ,					self.num_labels     )
														__A	:						Tuple       =				ids_tensor([self.batch_size]     ,					self.num_choices     )

									__A	:						Optional[Any]       =				BertConfig(
									    vocab_size=self.vocab_size     ,					hidden_size=self.hidden_size     ,					num_hidden_layers=self.num_hidden_layers     ,					num_attention_heads=self.num_attention_heads     ,					intermediate_size=self.intermediate_size     ,					hidden_act=self.hidden_act     ,					hidden_dropout_prob=self.hidden_dropout_prob     ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,					max_position_embeddings=self.max_position_embeddings     ,					type_vocab_size=self.type_vocab_size     ,					is_decoder=_A     ,					initializer_range=self.initializer_range     ,					)
									__A	:						List[str]       =				DPRConfig(projection_dim=self.projection_dim     ,					**config.to_dict()     )

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



				def 					UpperCAmelCase_		(			self     ,					_A     ,					_A     ,					_A     ,					_A     ,					_A     ,					_A     ,					_A     ):
									__A	:						Dict       =				TFDPRContextEncoder(config=_A     )
									__A	:						str       =				model(_A     ,					attention_mask=_A     ,					token_type_ids=_A     )
									__A	:						Optional[Any]       =				model(_A     ,					token_type_ids=_A     )
									__A	:						int       =				model(_A     )
									self.parent.assertEqual(result.pooler_output.shape     ,					(self.batch_size, self.projection_dim or self.hidden_size)     )



				def 					UpperCAmelCase_		(			self     ,					_A     ,					_A     ,					_A     ,					_A     ,					_A     ,					_A     ,					_A     ):
									__A	:						List[str]       =				TFDPRQuestionEncoder(config=_A     )
									__A	:						Optional[Any]       =				model(_A     ,					attention_mask=_A     ,					token_type_ids=_A     )
									__A	:						Any       =				model(_A     ,					token_type_ids=_A     )
									__A	:						Dict       =				model(_A     )
									self.parent.assertEqual(result.pooler_output.shape     ,					(self.batch_size, self.projection_dim or self.hidden_size)     )



				def 					UpperCAmelCase_		(			self     ,					_A     ,					_A     ,					_A     ,					_A     ,					_A     ,					_A     ,					_A     ):
									__A	:						int       =				TFDPRReader(config=_A     )
									__A	:						List[str]       =				model(_A     ,					attention_mask=_A     )

									self.parent.assertEqual(result.start_logits.shape     ,					(self.batch_size, self.seq_length)     )
									self.parent.assertEqual(result.end_logits.shape     ,					(self.batch_size, self.seq_length)     )
									self.parent.assertEqual(result.relevance_logits.shape     ,					(self.batch_size,)     )





				def 					UpperCAmelCase_		(			self     ):
									__A	:						Optional[Any]       =				self.prepare_config_and_inputs()
									(
									    __A
									)	:						Optional[int]       =				config_and_inputs
									__A	:						Optional[Any]       =				{"""input_ids""": input_ids}
									return config, inputs_dict





@require_tf
class 						_A(					snake_case__				, snake_case__				, unittest.TestCase   ):


				"""simple docstring"""


				UpperCamelCase    :  Optional[Any]					    =	(
				    (
				        TFDPRContextEncoder,
				        TFDPRQuestionEncoder,
				        TFDPRReader,
				    )
				    if is_tf_available()
				    else ()
				)
				UpperCamelCase    :  Any					    =	{'''feature-extraction''': TFDPRQuestionEncoder} if is_tf_available() else {}

				UpperCamelCase    :  Optional[int]					    =	False
				UpperCamelCase    :  Tuple					    =	False
				UpperCamelCase    :  Optional[int]					    =	False
				UpperCamelCase    :  str					    =	False
				UpperCamelCase    :  List[Any]					    =	False



				def 					UpperCAmelCase_		(			self     ):
									__A	:						List[Any]       =				TFDPRModelTester(self     )
									__A	:						Union[str, Any]       =				ConfigTester(self     ,					config_class=_A     ,					hidden_size=37     )



				def 					UpperCAmelCase_		(			self     ):
									self.config_tester.run_common_tests()



				def 					UpperCAmelCase_		(			self     ):
									__A	:						List[Any]       =				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_dpr_context_encoder(*_A     )



				def 					UpperCAmelCase_		(			self     ):
									__A	:						List[Any]       =				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_dpr_question_encoder(*_A     )



				def 					UpperCAmelCase_		(			self     ):
									__A	:						List[Any]       =				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_dpr_reader(*_A     )





				@slow
				def 					UpperCAmelCase_		(			self     ):






									for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
														__A	:						Optional[Any]       =				TFDPRContextEncoder.from_pretrained(_A     )
														self.assertIsNotNone(_A     )

									for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
														__A	:						Tuple       =				TFDPRContextEncoder.from_pretrained(_A     )
														self.assertIsNotNone(_A     )

									for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
														__A	:						Dict       =				TFDPRQuestionEncoder.from_pretrained(_A     )
														self.assertIsNotNone(_A     )

									for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
														__A	:						str       =				TFDPRReader.from_pretrained(_A     )
														self.assertIsNotNone(_A     )





@require_tf
class 						_A(					unittest.TestCase   ):


				"""simple docstring"""


				@slow
				def 					UpperCAmelCase_		(			self     ):
									__A	:						Dict       =				TFDPRQuestionEncoder.from_pretrained('facebook/dpr-question_encoder-single-nq-base'     )

									__A	:						Any       =				tf.constant(
									    [[101, 7592, 1010, 2003, 2026, 3899, 10140, 1029, 102]]     )  # [CLS] hello, is my dog cute? [SEP]
									__A	:						Optional[Any]       =				model(_A     )[0]  # embedding shape = (1, 768)
									# compare the actual values for a slice.
									__A	:						Optional[Any]       =				tf.constant(
									    [
									        [
									            0.0_3_2_3_6_2_5_3,
									            0.1_2_7_5_3_3_3_5,
									            0.1_6_8_1_8_5_0_9,
									            0.0_0_2_7_9_7_8_6,
									            0.3_8_9_6_9_3_3,
									            0.2_4_2_6_4_9_4_5,
									            0.2_1_7_8_9_7_1,
									            -0.0_2_3_3_5_2_2_7,
									            -0.0_8_4_8_1_9_5_9,
									            -0.1_4_3_2_4_1_1_7,
									        ]
									    ]     )
									self.assertTrue(numpy.allclose(output[:, :10].numpy()     ,					expected_slice.numpy()     ,					atol=1e-4     )     )








"""simple docstring"""






import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class      lowerCamelCase__   (     A     ,    unittest.TestCase				):







					"""simple docstring"""


					__a	       =							None
					__a	       =							BloomTokenizerFast
					__a	       =							BloomTokenizerFast
					__a	       =							True
					__a	       =							False
					__a	       =							"""tokenizer_file"""
					__a	       =							{"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}






					def lowerCamelCase__							(       self    : int  ):
												'''simple docstring'''






												super().setUp()
												__UpperCAmelCase   :						Any      							=					BloomTokenizerFast.from_pretrained("""bigscience/tokenizer"""  )
												tokenizer.save_pretrained(self.tmpdirname  )






					def lowerCamelCase__							(       self    : Any ,       **UpperCamelCase    : Optional[int]  ):
												'''simple docstring'''






												kwargs.update(self.special_tokens_map  )
												return BloomTokenizerFast.from_pretrained(self.tmpdirname ,       **UpperCamelCase  )






					def lowerCamelCase__							(       self    : Optional[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					self.get_rust_tokenizer()

												__UpperCAmelCase   :						Optional[Any]      							=					["""The quick brown fox</s>""", """jumps over the lazy dog</s>"""]
												__UpperCAmelCase   :						int      							=					[[2_175, 23_714, 73_173, 144_252, 2], [77, 132_619, 3_478, 368, 109_586, 35_433, 2]]

												__UpperCAmelCase   :						Dict      							=					tokenizer.batch_encode_plus(UpperCamelCase  )["""input_ids"""]
												self.assertListEqual(UpperCamelCase ,       UpperCamelCase  )

												__UpperCAmelCase   :						int      							=					tokenizer.batch_decode(UpperCamelCase  )
												self.assertListEqual(UpperCamelCase ,       UpperCamelCase  )






					def lowerCamelCase__							(       self    : int ,       UpperCamelCase    : Any=6  ):
												'''simple docstring'''






												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																										__UpperCAmelCase   :						Optional[int]      							=					self.rust_tokenizer_class.from_pretrained(UpperCamelCase ,       **UpperCamelCase  )
																										# tokenizer_r.pad_token = None # Hotfixing padding = None
																										# Simple input
																										__UpperCAmelCase   :						Dict      							=					"""This is a simple input"""
																										__UpperCAmelCase   :						str      							=					["""This is a simple input 1""", """This is a simple input 2"""]
																										__UpperCAmelCase   :						List[str]      							=					("""This is a simple input""", """This is a pair""")
																										__UpperCAmelCase   :						Dict      							=					[
																										    ("""This is a simple input 1""", """This is a simple input 2"""),
																										    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																										]

																										# Simple input tests
																										try:
																																	tokenizer_r.encode(UpperCamelCase ,       max_length=UpperCamelCase  )
																																	tokenizer_r.encode_plus(UpperCamelCase ,       max_length=UpperCamelCase  )

																																	tokenizer_r.batch_encode_plus(UpperCamelCase ,       max_length=UpperCamelCase  )
																																	tokenizer_r.encode(UpperCamelCase ,       max_length=UpperCamelCase  )
																																	tokenizer_r.batch_encode_plus(UpperCamelCase ,       max_length=UpperCamelCase  )
																										except ValueError:
																																	self.fail("""Bloom Tokenizer should be able to deal with padding"""  )

																										__UpperCAmelCase   :						Union[str, Any]      							=					None  # Hotfixing padding = None
																										self.assertRaises(UpperCamelCase ,       tokenizer_r.encode ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length"""  )

																										# Simple input
																										self.assertRaises(UpperCamelCase ,       tokenizer_r.encode_plus ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length"""  )

																										# Simple input
																										self.assertRaises(
																										    UpperCamelCase ,       tokenizer_r.batch_encode_plus ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length""" ,       )

																										# Pair input
																										self.assertRaises(UpperCamelCase ,       tokenizer_r.encode ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length"""  )

																										# Pair input
																										self.assertRaises(UpperCamelCase ,       tokenizer_r.encode_plus ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length"""  )

																										# Pair input
																										self.assertRaises(
																										    UpperCamelCase ,       tokenizer_r.batch_encode_plus ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length""" ,       )






					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Union[str, Any]      							=					self.get_rust_tokenizer()
												__UpperCAmelCase   :						Optional[Any]      							=					load_dataset("""xnli""" ,       """all_languages""" ,       split="""test""" ,       streaming=UpperCamelCase  )

												__UpperCAmelCase   :						Optional[Any]      							=					next(iter(UpperCamelCase  )  )["""premise"""]  # pick up one data
												__UpperCAmelCase   :						Any      							=					list(sample_data.values()  )

												__UpperCAmelCase   :						Optional[Any]      							=					list(map(tokenizer.encode ,       UpperCamelCase  )  )
												__UpperCAmelCase   :						List[Any]      							=					[tokenizer.decode(UpperCamelCase ,       clean_up_tokenization_spaces=UpperCamelCase  ) for x in output_tokens]
												self.assertListEqual(UpperCamelCase ,       UpperCamelCase  )






					def lowerCamelCase__							(       self    : Optional[Any]  ):
												'''simple docstring'''






												self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map  ) ,       1  )
												self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()  )[0]  ) ,       1  )
