






import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin


class    __lowercase   (						lowercase_      ,   unittest.TestCase	):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE     =					DebertaTokenizer
     SCREAMING_SNAKE_CASE     =					True
     SCREAMING_SNAKE_CASE     =					DebertaTokenizerFast




     def 		lowerCAmelCase_  (					self							:      Optional[Any]					):


         """simple docstring"""



         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         __A       =						[
             """l""",
             """o""",
             """w""",
             """e""",
             """r""",
             """s""",
             """t""",
             """i""",
             """d""",
             """n""",
             """\u0120""",
             """\u0120l""",
             """\u0120n""",
             """\u0120lo""",
             """\u0120low""",
             """er""",
             """\u0120lowest""",
             """\u0120newer""",
             """\u0120wider""",
             """[UNK]""",
         ]
         __A       =						dict(zip(UpperCamelCase_			,						range(len(UpperCamelCase_					)					)					)					)
         __A       =						["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
         __A       =						{"""unk_token""": """[UNK]"""}

         __A       =						os.path.join(self.tmpdirname			,						VOCAB_FILES_NAMES["""vocab_file"""]					)
         __A       =						os.path.join(self.tmpdirname			,						VOCAB_FILES_NAMES["""merges_file"""]					)
         with open(self.vocab_file			,						"""w"""			,						encoding="""utf-8"""					) as fp:
             fp.write(json.dumps(UpperCamelCase_					) + """\n"""					)
         with open(self.merges_file			,						"""w"""			,						encoding="""utf-8"""					) as fp:
             fp.write("""\n""".join(UpperCamelCase_					)					)




     def 		lowerCAmelCase_  (					self							:      List[str]			,						**UpperCamelCase_							:      str					):


         """simple docstring"""



         kwargs.update(self.special_tokens_map					)
         return self.tokenizer_class.from_pretrained(self.tmpdirname			,						**UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      Any			,						UpperCamelCase_							:      int					):


         """simple docstring"""



         __A       =						"""lower newer"""
         __A       =						"""lower newer"""
         return input_text, output_text




     def 		lowerCAmelCase_  (					self							:      Tuple					):


         """simple docstring"""



         __A       =						self.get_tokenizer()
         __A       =						"""lower newer"""
         __A       =						["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
         __A       =						tokenizer.tokenize(UpperCamelCase_					)
         self.assertListEqual(UpperCamelCase_			,						UpperCamelCase_					)

         __A       =						tokens + [tokenizer.unk_token]
         __A       =						[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_					)			,						UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      Optional[Any]					):


         """simple docstring"""



         __A       =						self.get_tokenizer()
         __A       =						tokenizer("""Hello"""			,						"""World"""					)
         __A       =						[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
         self.assertListEqual(tokd["""token_type_ids"""]			,						UpperCamelCase_					)




     @slow
     def 		lowerCAmelCase_  (					self							:      Tuple					):


         """simple docstring"""



         __A       =						self.tokenizer_class.from_pretrained("""microsoft/deberta-base"""					)

         __A       =						tokenizer.encode("""sequence builders"""			,						add_special_tokens=UpperCamelCase_					)
         __A       =						tokenizer.encode("""multi-sequence build"""			,						add_special_tokens=UpperCamelCase_					)

         __A       =						tokenizer.encode(
             """sequence builders"""			,						add_special_tokens=UpperCamelCase_			,						add_prefix_space=UpperCamelCase_					)
         __A       =						tokenizer.encode(
             """sequence builders"""			,						"""multi-sequence build"""			,						add_special_tokens=UpperCamelCase_			,						add_prefix_space=UpperCamelCase_					)

         __A       =						tokenizer.build_inputs_with_special_tokens(UpperCamelCase_					)
         __A       =						tokenizer.build_inputs_with_special_tokens(UpperCamelCase_			,						UpperCamelCase_					)

         assert encoded_sentence == encoded_text_from_decode
         assert encoded_pair == encoded_pair_from_decode





     @slow
     def 		lowerCAmelCase_  (					self							:      Dict					):


         """simple docstring"""



         __A       =						[self.tokenizer_class]
         if self.test_rust_tokenizer:
             tokenizer_classes.append(self.rust_tokenizer_class					)

         for tokenizer_class in tokenizer_classes:
             __A       =						tokenizer_class.from_pretrained("""microsoft/deberta-base"""					)

             __A       =						[
                 """ALBERT: A Lite BERT for Self-supervised Learning of Language Representations""",
                 """ALBERT incorporates two parameter reduction techniques""",
                 """The first one is a factorized embedding parameterization. By decomposing the large vocabulary"""
                 """ embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"""
                 """ vocabulary embedding.""",
             ]

             __A       =						tokenizer(UpperCamelCase_			,						padding=UpperCamelCase_					)
             __A       =						[tokenizer.decode(UpperCamelCase_			,						skip_special_tokens=UpperCamelCase_					) for seq in encoding["""input_ids"""]]

             # fmt: off
             __A       =						{
                 """input_ids""": [
                     [1, 2_118, 11_126, 565, 35, 83, 25_191, 163, 18_854, 13, 12_156, 12, 16_101, 25_376, 13_807, 9, 22_205, 27_893, 1_635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 2_118, 11_126, 565, 24_536, 80, 43_797, 4_878, 7_373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 133, 78, 65, 16, 10, 3_724, 1_538, 33_183, 11_303, 43_797, 1_938, 4, 870, 24_165, 29_105, 5, 739, 32_644, 33_183, 11_303, 36_173, 88, 80, 650, 7_821, 45_940, 6, 52, 2_559, 5, 1_836, 9, 5, 7_397, 13_171, 31, 5, 1_836, 9, 32_644, 33_183, 11_303, 4, 2]
                 ],
                 """token_type_ids""": [
                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                 ],
                 """attention_mask""": [
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
                 ]
             }
             # fmt: on

             __A       =						[
                 """ALBERT: A Lite BERT for Self-supervised Learning of Language Representations""",
                 """ALBERT incorporates two parameter reduction techniques""",
                 """The first one is a factorized embedding parameterization. By decomposing the large vocabulary"""
                 """ embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"""
                 """ vocabulary embedding.""",
             ]

             self.assertDictEqual(encoding.data			,						UpperCamelCase_					)

             for expected, decoded in zip(UpperCamelCase_			,						UpperCamelCase_					):
                 self.assertEqual(UpperCamelCase_			,						UpperCamelCase_					)












from ...configuration_utils import PretrainedConfig
from ...utils import logging


__a   :							Optional[Any]						= logging.get_logger(__name__)

__a   :							Dict						= {
    "google/realm-cc-news-pretrained-embedder": (
        "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/config.json"
    ),
    "google/realm-cc-news-pretrained-encoder": (
        "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/config.json"
    ),
    "google/realm-cc-news-pretrained-scorer": (
        "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/config.json"
    ),
    "google/realm-cc-news-pretrained-openqa": (
        "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/config.json"
    ),
    "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/config.json",
    "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/config.json",
    "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/config.json",
    "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/config.json",
    # See all REALM models at https://huggingface.co/models?filter=realm
}


class    __lowercase   (						lowercase_	):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE     =					"realm"




     def __init__(					self							:      str			,						UpperCamelCase_							:      List[Any]=30_522			,						UpperCamelCase_							:      Dict=768			,						UpperCamelCase_							:      Union[str, Any]=128			,						UpperCamelCase_							:      Tuple=12			,						UpperCamelCase_							:      Optional[Any]=12			,						UpperCamelCase_							:      Optional[int]=8			,						UpperCamelCase_							:      str=3_072			,						UpperCamelCase_							:      List[str]="gelu_new"			,						UpperCamelCase_							:      Optional[int]=0.1			,						UpperCamelCase_							:      Optional[int]=0.1			,						UpperCamelCase_							:      str=512			,						UpperCamelCase_							:      int=2			,						UpperCamelCase_							:      str=0.02			,						UpperCamelCase_							:      List[Any]=1e-12			,						UpperCamelCase_							:      str=256			,						UpperCamelCase_							:      List[str]=10			,						UpperCamelCase_							:      int=1e-3			,						UpperCamelCase_							:      List[str]=5			,						UpperCamelCase_							:      Tuple=320			,						UpperCamelCase_							:      List[str]=13_353_718			,						UpperCamelCase_							:      Tuple=5_000			,						UpperCamelCase_							:      Optional[int]=1			,						UpperCamelCase_							:      List[str]=0			,						UpperCamelCase_							:      Union[str, Any]=2			,						**UpperCamelCase_							:      List[Any]			,						):


         """simple docstring"""



         super().__init__(pad_token_id=UpperCamelCase_			,						bos_token_id=UpperCamelCase_			,						eos_token_id=UpperCamelCase_			,						**UpperCamelCase_					)

         # Common config
         __A       =						vocab_size
         __A       =						max_position_embeddings
         __A       =						hidden_size
         __A       =						retriever_proj_size
         __A       =						num_hidden_layers
         __A       =						num_attention_heads
         __A       =						num_candidates
         __A       =						intermediate_size
         __A       =						hidden_act
         __A       =						hidden_dropout_prob
         __A       =						attention_probs_dropout_prob
         __A       =						initializer_range
         __A       =						type_vocab_size
         __A       =						layer_norm_eps

         # Reader config
         __A       =						span_hidden_size
         __A       =						max_span_width
         __A       =						reader_layer_norm_eps
         __A       =						reader_beam_size
         __A       =						reader_seq_len

         # Retrieval config
         __A       =						num_block_records
         __A       =						searcher_beam_size




