


"""simple docstring"""






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import numpy
							import tensorflow as tf

							from transformers import (
							    TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
							    TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
							    TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
							    BertConfig,
							    DPRConfig,
							    TFDPRContextEncoder,
							    TFDPRQuestionEncoder,
							    TFDPRReader,
							)





class      lowerCamelCase__   :







					"""simple docstring"""


					def __init__(       self    : Any ,       UpperCamelCase    : Any ,       UpperCamelCase    : Dict=13 ,       UpperCamelCase    : List[Any]=7 ,       UpperCamelCase    : List[Any]=True ,       UpperCamelCase    : Union[str, Any]=True ,       UpperCamelCase    : List[Any]=True ,       UpperCamelCase    : Dict=True ,       UpperCamelCase    : List[Any]=99 ,       UpperCamelCase    : Optional[int]=32 ,       UpperCamelCase    : int=2 ,       UpperCamelCase    : str=4 ,       UpperCamelCase    : List[Any]=37 ,       UpperCamelCase    : Dict="gelu" ,       UpperCamelCase    : str=0.1 ,       UpperCamelCase    : List[str]=0.1 ,       UpperCamelCase    : str=512 ,       UpperCamelCase    : str=16 ,       UpperCamelCase    : Optional[Any]=2 ,       UpperCamelCase    : Any=0.02 ,       UpperCamelCase    : List[Any]=3 ,       UpperCamelCase    : List[str]=4 ,       UpperCamelCase    : List[str]=None ,       UpperCamelCase    : Optional[int]=0 ,       ):
												'''simple docstring'''






												__UpperCAmelCase   :						Dict      							=					parent
												__UpperCAmelCase   :						List[str]      							=					batch_size
												__UpperCAmelCase   :						int      							=					seq_length
												__UpperCAmelCase   :						List[str]      							=					is_training
												__UpperCAmelCase   :						List[Any]      							=					use_input_mask
												__UpperCAmelCase   :						Dict      							=					use_token_type_ids
												__UpperCAmelCase   :						List[Any]      							=					use_labels
												__UpperCAmelCase   :						List[Any]      							=					vocab_size
												__UpperCAmelCase   :						Tuple      							=					hidden_size
												__UpperCAmelCase   :						Union[str, Any]      							=					num_hidden_layers
												__UpperCAmelCase   :						int      							=					num_attention_heads
												__UpperCAmelCase   :						Tuple      							=					intermediate_size
												__UpperCAmelCase   :						str      							=					hidden_act
												__UpperCAmelCase   :						Optional[int]      							=					hidden_dropout_prob
												__UpperCAmelCase   :						Optional[int]      							=					attention_probs_dropout_prob
												__UpperCAmelCase   :						int      							=					max_position_embeddings
												__UpperCAmelCase   :						Dict      							=					type_vocab_size
												__UpperCAmelCase   :						int      							=					type_sequence_label_size
												__UpperCAmelCase   :						Optional[int]      							=					initializer_range
												__UpperCAmelCase   :						Any      							=					num_labels
												__UpperCAmelCase   :						Any      							=					num_choices
												__UpperCAmelCase   :						int      							=					scope
												__UpperCAmelCase   :						Tuple      							=					projection_dim






					def lowerCamelCase__							(       self    : Optional[int]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					ids_tensor([self.batch_size, self.seq_length] ,       self.vocab_size  )

												__UpperCAmelCase   :						Any      							=					None
												if self.use_input_mask:
																			# follow test_modeling_tf_ctrl.py
																			__UpperCAmelCase   :						List[Any]      							=					random_attention_mask([self.batch_size, self.seq_length]  )

												__UpperCAmelCase   :						int      							=					None
												if self.use_token_type_ids:
																			__UpperCAmelCase   :						Union[str, Any]      							=					ids_tensor([self.batch_size, self.seq_length] ,       self.type_vocab_size  )

												__UpperCAmelCase   :						str      							=					None
												__UpperCAmelCase   :						Tuple      							=					None
												__UpperCAmelCase   :						Optional[Any]      							=					None
												if self.use_labels:
																			__UpperCAmelCase   :						Dict      							=					ids_tensor([self.batch_size] ,       self.type_sequence_label_size  )
																			__UpperCAmelCase   :						List[Any]      							=					ids_tensor([self.batch_size, self.seq_length] ,       self.num_labels  )
																			__UpperCAmelCase   :						Tuple      							=					ids_tensor([self.batch_size] ,       self.num_choices  )

												__UpperCAmelCase   :						Optional[Any]      							=					BertConfig(
												    vocab_size=self.vocab_size ,       hidden_size=self.hidden_size ,       num_hidden_layers=self.num_hidden_layers ,       num_attention_heads=self.num_attention_heads ,       intermediate_size=self.intermediate_size ,       hidden_act=self.hidden_act ,       hidden_dropout_prob=self.hidden_dropout_prob ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob ,       max_position_embeddings=self.max_position_embeddings ,       type_vocab_size=self.type_vocab_size ,       is_decoder=UpperCamelCase ,       initializer_range=self.initializer_range ,       )
												__UpperCAmelCase   :						List[str]      							=					DPRConfig(projection_dim=self.projection_dim ,       **config.to_dict()  )

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






					def lowerCamelCase__							(       self    : List[Any] ,       UpperCamelCase    : Tuple ,       UpperCamelCase    : int ,       UpperCamelCase    : Optional[Any] ,       UpperCamelCase    : Dict ,       UpperCamelCase    : Dict ,       UpperCamelCase    : int ,       UpperCamelCase    : List[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Dict      							=					TFDPRContextEncoder(config=UpperCamelCase  )
												__UpperCAmelCase   :						str      							=					model(UpperCamelCase ,       attention_mask=UpperCamelCase ,       token_type_ids=UpperCamelCase  )
												__UpperCAmelCase   :						Optional[Any]      							=					model(UpperCamelCase ,       token_type_ids=UpperCamelCase  )
												__UpperCAmelCase   :						int      							=					model(UpperCamelCase  )
												self.parent.assertEqual(result.pooler_output.shape ,       (self.batch_size, self.projection_dim or self.hidden_size)  )






					def lowerCamelCase__							(       self    : int ,       UpperCamelCase    : Optional[Any] ,       UpperCamelCase    : Union[str, Any] ,       UpperCamelCase    : Any ,       UpperCamelCase    : str ,       UpperCamelCase    : Tuple ,       UpperCamelCase    : List[str] ,       UpperCamelCase    : Dict  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					TFDPRQuestionEncoder(config=UpperCamelCase  )
												__UpperCAmelCase   :						Optional[Any]      							=					model(UpperCamelCase ,       attention_mask=UpperCamelCase ,       token_type_ids=UpperCamelCase  )
												__UpperCAmelCase   :						Any      							=					model(UpperCamelCase ,       token_type_ids=UpperCamelCase  )
												__UpperCAmelCase   :						Dict      							=					model(UpperCamelCase  )
												self.parent.assertEqual(result.pooler_output.shape ,       (self.batch_size, self.projection_dim or self.hidden_size)  )






					def lowerCamelCase__							(       self    : Optional[Any] ,       UpperCamelCase    : str ,       UpperCamelCase    : Tuple ,       UpperCamelCase    : str ,       UpperCamelCase    : List[str] ,       UpperCamelCase    : int ,       UpperCamelCase    : Optional[Any] ,       UpperCamelCase    : Dict  ):
												'''simple docstring'''






												__UpperCAmelCase   :						int      							=					TFDPRReader(config=UpperCamelCase  )
												__UpperCAmelCase   :						List[str]      							=					model(UpperCamelCase ,       attention_mask=UpperCamelCase  )

												self.parent.assertEqual(result.start_logits.shape ,       (self.batch_size, self.seq_length)  )
												self.parent.assertEqual(result.end_logits.shape ,       (self.batch_size, self.seq_length)  )
												self.parent.assertEqual(result.relevance_logits.shape ,       (self.batch_size,)  )






					def lowerCamelCase__							(       self    : Tuple  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Optional[Any]      							=					self.prepare_config_and_inputs()
												(
												    (
												    __UpperCAmelCase
												)							,(
												    __UpperCAmelCase
												)							,(
												    __UpperCAmelCase
												)							,(
												    __UpperCAmelCase
												)							,(
												    __UpperCAmelCase
												)							,(
												    __UpperCAmelCase
												)							,(
												    __UpperCAmelCase
												)							,
												)   :						Optional[int]      							=					config_and_inputs
												__UpperCAmelCase   :						Optional[Any]      							=					{"""input_ids""": input_ids}
												return config, inputs_dict






@require_tf
class      lowerCamelCase__   (     A     ,    A     ,    unittest.TestCase				):







					"""simple docstring"""


					__a	       =							(
					    (
					        TFDPRContextEncoder,
					        TFDPRQuestionEncoder,
					        TFDPRReader,
					    )
					    if is_tf_available()
					    else ()
					)
					__a	       =							{"""feature-extraction""": TFDPRQuestionEncoder} if is_tf_available() else {}

					__a	       =							False
					__a	       =							False
					__a	       =							False
					__a	       =							False
					__a	       =							False






					def lowerCamelCase__							(       self    : Optional[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[Any]      							=					TFDPRModelTester(self  )
												__UpperCAmelCase   :						Union[str, Any]      							=					ConfigTester(self ,       config_class=UpperCamelCase ,       hidden_size=37  )






					def lowerCamelCase__							(       self    : Tuple  ):
												'''simple docstring'''






												self.config_tester.run_common_tests()






					def lowerCamelCase__							(       self    : Optional[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[Any]      							=					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_dpr_context_encoder(*UpperCamelCase  )






					def lowerCamelCase__							(       self    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[Any]      							=					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_dpr_question_encoder(*UpperCamelCase  )






					def lowerCamelCase__							(       self    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[Any]      							=					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_dpr_reader(*UpperCamelCase  )






					@slow
					def lowerCamelCase__							(       self    : Dict  ):
												'''simple docstring'''






												for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																			__UpperCAmelCase   :						Optional[Any]      							=					TFDPRContextEncoder.from_pretrained(UpperCamelCase  )
																			self.assertIsNotNone(UpperCamelCase  )

												for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																			__UpperCAmelCase   :						Tuple      							=					TFDPRContextEncoder.from_pretrained(UpperCamelCase  )
																			self.assertIsNotNone(UpperCamelCase  )

												for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																			__UpperCAmelCase   :						Dict      							=					TFDPRQuestionEncoder.from_pretrained(UpperCamelCase  )
																			self.assertIsNotNone(UpperCamelCase  )

												for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																			__UpperCAmelCase   :						str      							=					TFDPRReader.from_pretrained(UpperCamelCase  )
																			self.assertIsNotNone(UpperCamelCase  )






@require_tf
class      lowerCamelCase__   (     unittest.TestCase				):







					"""simple docstring"""


					@slow
					def lowerCamelCase__							(       self    : int  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Dict      							=					TFDPRQuestionEncoder.from_pretrained("""facebook/dpr-question_encoder-single-nq-base"""  )

												__UpperCAmelCase   :						Any      							=					tf.constant(
												    [[101, 7_592, 1_010, 2_003, 2_026, 3_899, 10_140, 1_029, 102]]  )  # [CLS] hello, is my dog cute? [SEP]
												__UpperCAmelCase   :						Optional[Any]      							=					model(UpperCamelCase  )[0]  # embedding shape = (1, 768)
												# compare the actual values for a slice.
												__UpperCAmelCase   :						Optional[Any]      							=					tf.constant(
												    [
												        [
												            0.03236253,
												            0.12753335,
												            0.16818509,
												            0.00279786,
												            0.3896933,
												            0.24264945,
												            0.2178971,
												            -0.02335227,
												            -0.08481959,
												            -0.14324117,
												        ]
												    ]  )
												self.assertTrue(numpy.allclose(output[:, :10].numpy() ,       expected_slice.numpy() ,       atol=1e-4  )  )




"""simple docstring"""






import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class      lowerCamelCase__   (     A     ,    unittest.TestCase				):







					"""simple docstring"""


					__a	       =							None
					__a	       =							BloomTokenizerFast
					__a	       =							BloomTokenizerFast
					__a	       =							True
					__a	       =							False
					__a	       =							"""tokenizer_file"""
					__a	       =							{"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}






					def lowerCamelCase__							(       self    : int  ):
												'''simple docstring'''






												super().setUp()
												__UpperCAmelCase   :						Any      							=					BloomTokenizerFast.from_pretrained("""bigscience/tokenizer"""  )
												tokenizer.save_pretrained(self.tmpdirname  )






					def lowerCamelCase__							(       self    : Any ,       **UpperCamelCase    : Optional[int]  ):
												'''simple docstring'''






												kwargs.update(self.special_tokens_map  )
												return BloomTokenizerFast.from_pretrained(self.tmpdirname ,       **UpperCamelCase  )






					def lowerCamelCase__							(       self    : Optional[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					self.get_rust_tokenizer()

												__UpperCAmelCase   :						Optional[Any]      							=					["""The quick brown fox</s>""", """jumps over the lazy dog</s>"""]
												__UpperCAmelCase   :						int      							=					[[2_175, 23_714, 73_173, 144_252, 2], [77, 132_619, 3_478, 368, 109_586, 35_433, 2]]

												__UpperCAmelCase   :						Dict      							=					tokenizer.batch_encode_plus(UpperCamelCase  )["""input_ids"""]
												self.assertListEqual(UpperCamelCase ,       UpperCamelCase  )

												__UpperCAmelCase   :						int      							=					tokenizer.batch_decode(UpperCamelCase  )
												self.assertListEqual(UpperCamelCase ,       UpperCamelCase  )






					def lowerCamelCase__							(       self    : int ,       UpperCamelCase    : Any=6  ):
												'''simple docstring'''






												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																										__UpperCAmelCase   :						Optional[int]      							=					self.rust_tokenizer_class.from_pretrained(UpperCamelCase ,       **UpperCamelCase  )
																										# tokenizer_r.pad_token = None # Hotfixing padding = None
																										# Simple input
																										__UpperCAmelCase   :						Dict      							=					"""This is a simple input"""
																										__UpperCAmelCase   :						str      							=					["""This is a simple input 1""", """This is a simple input 2"""]
																										__UpperCAmelCase   :						List[str]      							=					("""This is a simple input""", """This is a pair""")
																										__UpperCAmelCase   :						Dict      							=					[
																										    ("""This is a simple input 1""", """This is a simple input 2"""),
																										    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																										]

																										# Simple input tests
																										try:
																																	tokenizer_r.encode(UpperCamelCase ,       max_length=UpperCamelCase  )
																																	tokenizer_r.encode_plus(UpperCamelCase ,       max_length=UpperCamelCase  )

																																	tokenizer_r.batch_encode_plus(UpperCamelCase ,       max_length=UpperCamelCase  )
																																	tokenizer_r.encode(UpperCamelCase ,       max_length=UpperCamelCase  )
																																	tokenizer_r.batch_encode_plus(UpperCamelCase ,       max_length=UpperCamelCase  )
																										except ValueError:
																																	self.fail("""Bloom Tokenizer should be able to deal with padding"""  )

																										__UpperCAmelCase   :						Union[str, Any]      							=					None  # Hotfixing padding = None
																										self.assertRaises(UpperCamelCase ,       tokenizer_r.encode ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length"""  )

																										# Simple input
																										self.assertRaises(UpperCamelCase ,       tokenizer_r.encode_plus ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length"""  )

																										# Simple input
																										self.assertRaises(
																										    UpperCamelCase ,       tokenizer_r.batch_encode_plus ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length""" ,       )

																										# Pair input
																										self.assertRaises(UpperCamelCase ,       tokenizer_r.encode ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length"""  )

																										# Pair input
																										self.assertRaises(UpperCamelCase ,       tokenizer_r.encode_plus ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length"""  )

																										# Pair input
																										self.assertRaises(
																										    UpperCamelCase ,       tokenizer_r.batch_encode_plus ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length""" ,       )






					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Union[str, Any]      							=					self.get_rust_tokenizer()
												__UpperCAmelCase   :						Optional[Any]      							=					load_dataset("""xnli""" ,       """all_languages""" ,       split="""test""" ,       streaming=UpperCamelCase  )

												__UpperCAmelCase   :						Optional[Any]      							=					next(iter(UpperCamelCase  )  )["""premise"""]  # pick up one data
												__UpperCAmelCase   :						Any      							=					list(sample_data.values()  )

												__UpperCAmelCase   :						Optional[Any]      							=					list(map(tokenizer.encode ,       UpperCamelCase  )  )
												__UpperCAmelCase   :						List[Any]      							=					[tokenizer.decode(UpperCamelCase ,       clean_up_tokenization_spaces=UpperCamelCase  ) for x in output_tokens]
												self.assertListEqual(UpperCamelCase ,       UpperCamelCase  )






					def lowerCamelCase__							(       self    : Optional[Any]  ):
												'''simple docstring'''






												self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map  ) ,       1  )
												self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()  )[0]  ) ,       1  )
