




import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
      import tensorflow as tf

if is_tensorflow_text_available():
      from transformers.models.bert import TFBertTokenizer


__magic_name__  :  Tuple					=       ["""bert-base-uncased""", """bert-base-cased"""]
__magic_name__  :  Dict					=       """hf-internal-testing/tiny-bert-tf-only"""

if is_tf_available():





      class     SCREAMING_SNAKE_CASE__		(tf.keras.Model			):
           def __init__(						self	:		int       ,						__lowerCamelCase	:		List[str]						):



             """simple docstring"""
             super().__init__()
             lowerCAmelCase__										=      tokenizer
             lowerCAmelCase__										=      AutoConfig.from_pretrained(__lowerCamelCase						)
             lowerCAmelCase__										=      TFAutoModel.from_config(__lowerCamelCase						)



           def        A__      (						self	:		Tuple       ,						__lowerCamelCase	:		Any						):



             """simple docstring"""
             lowerCAmelCase__										=      self.tokenizer(__lowerCamelCase						)
             lowerCAmelCase__										=      self.bert(**__lowerCamelCase						)
             return out["pooler_output"]





@require_tf
@require_tensorflow_text
class     SCREAMING_SNAKE_CASE__		(unittest.TestCase			):
     def        A__      (						self	:		Any						):



       """simple docstring"""
       super().setUp()

       lowerCAmelCase__										=      [
           BertTokenizer.from_pretrained(__lowerCamelCase						) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
       ]  # repeat for when fast_bert_tokenizer=false
       lowerCAmelCase__										=      [TFBertTokenizer.from_pretrained(__lowerCamelCase						) for checkpoint in TOKENIZER_CHECKPOINTS] + [
           TFBertTokenizer.from_pretrained(__lowerCamelCase       ,						use_fast_bert_tokenizer=__lowerCamelCase						)
           for checkpoint in TOKENIZER_CHECKPOINTS
       ]
       assert len(self.tokenizers						) == len(self.tf_tokenizers						)

       lowerCAmelCase__										=      [
           '''This is a straightforward English test sentence.''',
           '''This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.''',
           '''Now we\'re going to add some Chinese: 一 二 三 一二三''',
           '''And some much more rare Chinese: 齉 堃 齉堃''',
           '''Je vais aussi écrire en français pour tester les accents''',
           '''Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ''',
       ]
       lowerCAmelCase__										=      list(zip(self.test_sentences       ,						self.test_sentences[::-1]						)						)
     def        A__      (						self	:		int						):



       """simple docstring"""
       for tokenizer, tf_tokenizer in zip(self.tokenizers       ,						self.tf_tokenizers						):
         for test_inputs in (self.test_sentences, self.paired_sentences):
           lowerCAmelCase__										=      tokenizer(__lowerCamelCase       ,						return_tensors='''tf'''       ,						padding='''longest'''						)
           lowerCAmelCase__										=      tf_tokenizer(__lowerCamelCase						)

           for key in python_outputs.keys():
             self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape						)						)
             self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key]       ,						tf.intaa						) == tf_outputs[key]						)						)
     @slow
     def        A__      (						self	:		Optional[int]						):



       """simple docstring"""
       for tf_tokenizer in self.tf_tokenizers:
         lowerCAmelCase__										=      tf_tokenizer(self.paired_sentences						)
         lowerCAmelCase__										=      tf_tokenizer(
             text=[sentence[0] for sentence in self.paired_sentences]       ,						text_pair=[sentence[1] for sentence in self.paired_sentences]       ,						)
         for key in merged_outputs.keys():
           self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key]       ,						tf.intaa						) == separated_outputs[key]						)						)
     @slow
     def        A__      (						self	:		Optional[int]						):



       """simple docstring"""
       for tf_tokenizer in self.tf_tokenizers:
         lowerCAmelCase__										=      tf.function(__lowerCamelCase						)
         for test_inputs in (self.test_sentences, self.paired_sentences):
           lowerCAmelCase__										=      tf.constant(__lowerCamelCase						)
           lowerCAmelCase__										=      compiled_tokenizer(__lowerCamelCase						)
           lowerCAmelCase__										=      tf_tokenizer(__lowerCamelCase						)

           for key in eager_outputs.keys():
             self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]						)						)



     @slow
     def        A__      (						self	:		Any						):



       """simple docstring"""
       for tf_tokenizer in self.tf_tokenizers:
         lowerCAmelCase__										=      ModelToSave(tokenizer=__lowerCamelCase						)
         lowerCAmelCase__										=      tf.convert_to_tensor(self.test_sentences						)
         lowerCAmelCase__										=      model(__lowerCamelCase						)  # Build model with some sample inputs
         with TemporaryDirectory() as tempdir:
           lowerCAmelCase__										=      Path(__lowerCamelCase						) / '''saved.model'''
           model.save(__lowerCamelCase						)
           lowerCAmelCase__										=      tf.keras.models.load_model(__lowerCamelCase						)
         lowerCAmelCase__										=      loaded_model(__lowerCamelCase						)
         # We may see small differences because the loaded model is compiled, so we need an epsilon for the test
         self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output						)						)       ,						1e-5						)







import logging

from transformers.configuration_utils import PretrainedConfig


__magic_name__  :  Tuple					=       logging.getLogger(__name__)





class     SCREAMING_SNAKE_CASE__		(_a			):
     lowercase_ :      Tuple				= "masked_bert"
     def __init__(						self	:		List[Any]       ,						__lowerCamelCase	:		Optional[Any]=3_05_22       ,						__lowerCamelCase	:		Dict=7_68       ,						__lowerCamelCase	:		Tuple=12       ,						__lowerCamelCase	:		int=12       ,						__lowerCamelCase	:		Tuple=30_72       ,						__lowerCamelCase	:		List[Any]="gelu"       ,						__lowerCamelCase	:		Union[str, Any]=0.1       ,						__lowerCamelCase	:		Dict=0.1       ,						__lowerCamelCase	:		Optional[int]=5_12       ,						__lowerCamelCase	:		List[Any]=2       ,						__lowerCamelCase	:		Union[str, Any]=0.02       ,						__lowerCamelCase	:		Optional[int]=1e-12       ,						__lowerCamelCase	:		Optional[int]=0       ,						__lowerCamelCase	:		List[str]="topK"       ,						__lowerCamelCase	:		Union[str, Any]="constant"       ,						__lowerCamelCase	:		Union[str, Any]=0.0       ,						**__lowerCamelCase	:		Tuple       ,						):



       """simple docstring"""
       super().__init__(pad_token_id=__lowerCamelCase       ,						**__lowerCamelCase						)

       lowerCAmelCase__										=      vocab_size
       lowerCAmelCase__										=      hidden_size
       lowerCAmelCase__										=      num_hidden_layers
       lowerCAmelCase__										=      num_attention_heads
       lowerCAmelCase__										=      hidden_act
       lowerCAmelCase__										=      intermediate_size
       lowerCAmelCase__										=      hidden_dropout_prob
       lowerCAmelCase__										=      attention_probs_dropout_prob
       lowerCAmelCase__										=      max_position_embeddings
       lowerCAmelCase__										=      type_vocab_size
       lowerCAmelCase__										=      initializer_range
       lowerCAmelCase__										=      layer_norm_eps
       lowerCAmelCase__										=      pruning_method
       lowerCAmelCase__										=      mask_init
       lowerCAmelCase__										=      mask_scale

