



'''simple docstring'''





import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin







class     UpperCAmelCase_       (       __SCREAMING_SNAKE_CASE		,		unittest.TestCase	):
    """simple docstring"""







    lowercase					=		DebertaTokenizer
    lowercase					=		True
    lowercase					=		DebertaTokenizerFast




    def lowerCamelCase       (    self					:     str						):
          super().setUp()

          # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
          snake_case__  :    Optional[int]    						=      [
              """l""",
              """o""",
              """w""",
              """e""",
              """r""",
              """s""",
              """t""",
              """i""",
              """d""",
              """n""",
              """\u0120""",
              """\u0120l""",
              """\u0120n""",
              """\u0120lo""",
              """\u0120low""",
              """er""",
              """\u0120lowest""",
              """\u0120newer""",
              """\u0120wider""",
              """[UNK]""",
          ]
          snake_case__  :    List[str]    						=      dict(zip(snake_case_ ,     range(len(snake_case_						)						)						)						)
          snake_case__  :    str    						=      ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
          snake_case__  :    Any    						=      {"""unk_token""": """[UNK]"""}

          snake_case__  :    List[Any]    						=      os.path.join(self.tmpdirname ,     VOCAB_FILES_NAMES["""vocab_file"""]						)
          snake_case__  :    Any    						=      os.path.join(self.tmpdirname ,     VOCAB_FILES_NAMES["""merges_file"""]						)
          with open(self.vocab_file ,     """w""" ,     encoding="""utf-8"""						) as fp:
                fp.write(json.dumps(snake_case_						) + """\n"""						)
          with open(self.merges_file ,     """w""" ,     encoding="""utf-8"""						) as fp:
                fp.write("""\n""".join(snake_case_						)						)




    def lowerCamelCase       (    self					:     int ,     **snake_case_					:     Dict						):
          kwargs.update(self.special_tokens_map						)
          return self.tokenizer_class.from_pretrained(self.tmpdirname ,     **snake_case_						)




    def lowerCamelCase       (    self					:     int ,     snake_case_					:     Optional[Any]						):
          snake_case__  :    int    						=      """lower newer"""
          snake_case__  :    List[Any]    						=      """lower newer"""
          return input_text, output_text




    def lowerCamelCase       (    self					:     List[str]						):
          snake_case__  :    Union[str, Any]    						=      self.get_tokenizer()
          snake_case__  :    Union[str, Any]    						=      """lower newer"""
          snake_case__  :    Union[str, Any]    						=      ["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
          snake_case__  :    Tuple    						=      tokenizer.tokenize(snake_case_						)
          self.assertListEqual(snake_case_ ,     snake_case_						)

          snake_case__  :    int    						=      tokens + [tokenizer.unk_token]
          snake_case__  :    Any    						=      [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_						) ,     snake_case_						)




    def lowerCamelCase       (    self					:     Dict						):
          snake_case__  :    Optional[Any]    						=      self.get_tokenizer()
          snake_case__  :    Optional[int]    						=      tokenizer("""Hello""" ,     """World"""						)
          snake_case__  :    Optional[int]    						=      [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
          self.assertListEqual(tokd["""token_type_ids"""] ,     snake_case_						)




    @slow
    def lowerCamelCase       (    self					:     List[str]						):
          snake_case__  :    Union[str, Any]    						=      self.tokenizer_class.from_pretrained("""microsoft/deberta-base"""						)

          snake_case__  :    Tuple    						=      tokenizer.encode("""sequence builders""" ,     add_special_tokens=snake_case_						)
          snake_case__  :    int    						=      tokenizer.encode("""multi-sequence build""" ,     add_special_tokens=snake_case_						)

          snake_case__  :    Optional[Any]    						=      tokenizer.encode(
              """sequence builders""" ,     add_special_tokens=snake_case_ ,     add_prefix_space=snake_case_						)
          snake_case__  :    Union[str, Any]    						=      tokenizer.encode(
              """sequence builders""" ,     """multi-sequence build""" ,     add_special_tokens=snake_case_ ,     add_prefix_space=snake_case_						)

          snake_case__  :    str    						=      tokenizer.build_inputs_with_special_tokens(snake_case_						)
          snake_case__  :    Optional[Any]    						=      tokenizer.build_inputs_with_special_tokens(snake_case_ ,     snake_case_						)

          assert encoded_sentence == encoded_text_from_decode
          assert encoded_pair == encoded_pair_from_decode




    @slow
    def lowerCamelCase       (    self					:     List[Any]						):
          snake_case__  :    str    						=      [self.tokenizer_class]
          if self.test_rust_tokenizer:
                tokenizer_classes.append(self.rust_tokenizer_class						)

          for tokenizer_class in tokenizer_classes:
                snake_case__  :    int    						=      tokenizer_class.from_pretrained("""microsoft/deberta-base"""						)

                snake_case__  :    List[Any]    						=      [
                    """ALBERT: A Lite BERT for Self-supervised Learning of Language Representations""",
                    """ALBERT incorporates two parameter reduction techniques""",
                    """The first one is a factorized embedding parameterization. By decomposing the large vocabulary"""
                    """ embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"""
                    """ vocabulary embedding.""",
                ]

                snake_case__  :    List[Any]    						=      tokenizer(snake_case_ ,     padding=snake_case_						)
                snake_case__  :    Any    						=      [tokenizer.decode(snake_case_ ,     skip_special_tokens=snake_case_						) for seq in encoding["""input_ids"""]]

                # fmt: off
                snake_case__  :    Union[str, Any]    						=      {
                    """input_ids""": [
                        [1, 2_118, 11_126, 565, 35, 83, 25_191, 163, 18_854, 13, 12_156, 12, 16_101, 25_376, 13_807, 9, 22_205, 27_893, 1_635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                        [1, 2_118, 11_126, 565, 24_536, 80, 43_797, 4_878, 7_373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                        [1, 133, 78, 65, 16, 10, 3_724, 1_538, 33_183, 11_303, 43_797, 1_938, 4, 870, 24_165, 29_105, 5, 739, 32_644, 33_183, 11_303, 36_173, 88, 80, 650, 7_821, 45_940, 6, 52, 2_559, 5, 1_836, 9, 5, 7_397, 13_171, 31, 5, 1_836, 9, 32_644, 33_183, 11_303, 4, 2]
                    ],
                    """token_type_ids""": [
                        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                    ],
                    """attention_mask""": [
                        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
                    ]
                }
                # fmt: on

                snake_case__  :    Any    						=      [
                    """ALBERT: A Lite BERT for Self-supervised Learning of Language Representations""",
                    """ALBERT incorporates two parameter reduction techniques""",
                    """The first one is a factorized embedding parameterization. By decomposing the large vocabulary"""
                    """ embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"""
                    """ vocabulary embedding.""",
                ]

                self.assertDictEqual(encoding.data ,     snake_case_						)

                for expected, decoded in zip(snake_case_ ,     snake_case_						):
                      self.assertEqual(snake_case_ ,     snake_case_						)


import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402


_lowerCamelCase					   ={
    """return_dict""": False,
    """output_hidden_states""": True,
    """output_attentions""": True,
    """torchscript""": True,
    """torch_dtype""": """float16""",
    """use_bfloat16""": True,
    """tf_legacy_loss""": True,
    """pruned_heads""": {"""a""": 1},
    """tie_word_embeddings""": False,
    """is_decoder""": True,
    """cross_attention_hidden_size""": 1_2_8,
    """add_cross_attention""": True,
    """tie_encoder_decoder""": True,
    """max_length""": 5_0,
    """min_length""": 3,
    """do_sample""": True,
    """early_stopping""": True,
    """num_beams""": 3,
    """num_beam_groups""": 3,
    """diversity_penalty""": 0.5,
    """temperature""": 2.0,
    """top_k""": 1_0,
    """top_p""": 0.7,
    """typical_p""": 0.2,
    """repetition_penalty""": 0.8,
    """length_penalty""": 0.8,
    """no_repeat_ngram_size""": 5,
    """encoder_no_repeat_ngram_size""": 5,
    """bad_words_ids""": [1, 2, 3],
    """num_return_sequences""": 3,
    """chunk_size_feed_forward""": 5,
    """output_scores""": True,
    """return_dict_in_generate""": True,
    """forced_bos_token_id""": 2,
    """forced_eos_token_id""": 3,
    """remove_invalid_values""": True,
    """architectures""": ["""BertModel"""],
    """finetuning_task""": """translation""",
    """id2label""": {0: """label"""},
    """label2id""": {"""label""": """0"""},
    """tokenizer_class""": """BertTokenizerFast""",
    """prefix""": """prefix""",
    """bos_token_id""": 6,
    """pad_token_id""": 7,
    """eos_token_id""": 8,
    """sep_token_id""": 9,
    """decoder_start_token_id""": 1_0,
    """exponential_decay_length_penalty""": (5, 1.01),
    """suppress_tokens""": [0, 1],
    """begin_suppress_tokens""": 2,
    """task_specific_params""": {"""translation""": """some_params"""},
    """problem_type""": """regression""",
}





@is_staging_test
class 							A__		(	unittest.TestCase):
      @classmethod
      def 						UpperCamelCase__							(      cls      ):
       lowerCamelCase		:	int							=      TOKEN
       HfFolder.save_token(__magic_name__      )
      @classmethod
      def 						UpperCamelCase__							(      cls      ):
       try:
        delete_repo(token=cls._token    ,  repo_id="""test-config"""      )
       except HTTPError:
        pass

       try:
        delete_repo(token=cls._token    ,  repo_id="""valid_org/test-config-org"""      )
       except HTTPError:
        pass

       try:
        delete_repo(token=cls._token    ,  repo_id="""test-dynamic-config"""      )
       except HTTPError:
        pass
      def 						UpperCamelCase__							(      self      ):
       lowerCamelCase		:	List[Any]							=      BertConfig(
           vocab_size=9_9    ,  hidden_size=3_2    ,  num_hidden_layers=5    ,  num_attention_heads=4    ,  intermediate_size=3_7      )
       config.push_to_hub("""test-config"""    ,  use_auth_token=self._token      )

       lowerCamelCase		:	Any							=      BertConfig.from_pretrained(F'''{USER}/test-config'''      )
       for k, v in config.to_dict().items():
        if k != "transformers_version":
         self.assertEqual(__magic_name__    ,  getattr(__magic_name__    ,  __magic_name__      )      )

        # Reset repo
       delete_repo(token=self._token    ,  repo_id="""test-config"""      )

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
        config.save_pretrained(__magic_name__    ,  repo_id="""test-config"""    ,  push_to_hub=__magic_name__    ,  use_auth_token=self._token      )

       lowerCamelCase		:	Optional[Any]							=      BertConfig.from_pretrained(F'''{USER}/test-config'''      )
       for k, v in config.to_dict().items():
        if k != "transformers_version":
         self.assertEqual(__magic_name__    ,  getattr(__magic_name__    ,  __magic_name__      )      )
      def 						UpperCamelCase__							(      self      ):
       lowerCamelCase		:	Dict							=      BertConfig(
           vocab_size=9_9    ,  hidden_size=3_2    ,  num_hidden_layers=5    ,  num_attention_heads=4    ,  intermediate_size=3_7      )
       config.push_to_hub("""valid_org/test-config-org"""    ,  use_auth_token=self._token      )

       lowerCamelCase		:	Optional[int]							=      BertConfig.from_pretrained("""valid_org/test-config-org"""      )
       for k, v in config.to_dict().items():
        if k != "transformers_version":
         self.assertEqual(__magic_name__    ,  getattr(__magic_name__    ,  __magic_name__      )      )

        # Reset repo
       delete_repo(token=self._token    ,  repo_id="""valid_org/test-config-org"""      )

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
        config.save_pretrained(
            __magic_name__    ,  repo_id="""valid_org/test-config-org"""    ,  push_to_hub=__magic_name__    ,  use_auth_token=self._token      )

       lowerCamelCase		:	List[str]							=      BertConfig.from_pretrained("""valid_org/test-config-org"""      )
       for k, v in config.to_dict().items():
        if k != "transformers_version":
         self.assertEqual(__magic_name__    ,  getattr(__magic_name__    ,  __magic_name__      )      )






      def 						UpperCamelCase__							(      self      ):
       CustomConfig.register_for_auto_class()
       lowerCamelCase		:	Optional[Any]							=      CustomConfig(attribute=4_2      )

       config.push_to_hub("""test-dynamic-config"""    ,  use_auth_token=self._token      )

       # This has added the proper auto_map field to the config
       self.assertDictEqual(config.auto_map    ,  {"""AutoConfig""": """custom_configuration.CustomConfig"""}      )

       lowerCamelCase		:	List[str]							=      AutoConfig.from_pretrained(F'''{USER}/test-dynamic-config'''    ,  trust_remote_code=__magic_name__      )
       # Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
       self.assertEqual(new_config.__class__.__name__    ,  """CustomConfig"""      )
       self.assertEqual(new_config.attribute    ,  4_2      )





class 							A__		(	unittest.TestCase):
      def 						UpperCamelCase__							(      self      ):
       lowerCamelCase		:	str							=      GPTaConfig()

       # attempt to modify each of int/float/bool/str config records and verify they were updated
       lowerCamelCase		:	Optional[int]							=      c.n_embd + 1  # int
       lowerCamelCase		:	Optional[int]							=      c.resid_pdrop + 1.0  # float
       lowerCamelCase		:	Tuple							=      not c.scale_attn_weights  # bool
       lowerCamelCase		:	Any							=      c.summary_type + """foo"""  # str
       c.update_from_string(
           F'''n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}'''      )
       self.assertEqual(__magic_name__    ,  c.n_embd    ,  """mismatch for key: n_embd"""      )
       self.assertEqual(__magic_name__    ,  c.resid_pdrop    ,  """mismatch for key: resid_pdrop"""      )
       self.assertEqual(__magic_name__    ,  c.scale_attn_weights    ,  """mismatch for key: scale_attn_weights"""      )
       self.assertEqual(__magic_name__    ,  c.summary_type    ,  """mismatch for key: summary_type"""      )
      def 						UpperCamelCase__							(      self      ):
       lowerCamelCase		:	str							=      PretrainedConfig()
       lowerCamelCase		:	int							=      [key for key in base_config.__dict__ if key not in config_common_kwargs]
       # If this part of the test fails, you have arguments to addin config_common_kwargs above.
       self.assertListEqual(
           __magic_name__    ,  ["""is_encoder_decoder""", """_name_or_path""", """_commit_hash""", """transformers_version"""]      )
       lowerCamelCase		:	List[str]							=      [key for key, value in config_common_kwargs.items() if value == getattr(__magic_name__    ,  __magic_name__      )]
       if len(__magic_name__      ) > 0:
        raise ValueError(
            """The following keys are set with the default values in"""
            """ `test_configuration_common.config_common_kwargs` pick another value for them:"""
            F''' {", ".join(__magic_name__      )}.'''      )
      def 						UpperCamelCase__							(      self      ):
       with self.assertRaises(__magic_name__      ):
        # config is in subfolder, the following should not work without specifying the subfolder
        lowerCamelCase		:	Dict							=      BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder"""      )

       lowerCamelCase		:	str							=      BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder"""    ,  subfolder="""bert"""      )

       self.assertIsNotNone(__magic_name__      )
      def 						UpperCamelCase__							(      self      ):
       # A mock response for an HTTP head request to emulate server down
       lowerCamelCase		:	Dict							=      mock.Mock()
       lowerCamelCase		:	Optional[int]							=      5_0_0
       lowerCamelCase		:	List[Any]							=      {}
       lowerCamelCase		:	Tuple							=      HTTPError
       lowerCamelCase		:	Union[str, Any]							=      {}

       # Download this model to make sure it's in the cache.
       lowerCamelCase		:	List[str]							=      BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""      )

       # Under the mock environment we get a 500 error when trying to reach the model.
       with mock.patch("""requests.Session.request"""    ,  return_value=__magic_name__      ) as mock_head:
        lowerCamelCase		:	Any							=      BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""      )
        # This check we did call the fake head request
        mock_head.assert_called()
      def 						UpperCamelCase__							(      self      ):
       # This test is for deprecated behavior and can be removed in v5
       lowerCamelCase		:	List[str]							=      BertConfig.from_pretrained(
           """https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json"""      )
      def 						UpperCamelCase__							(      self      ):
       lowerCamelCase		:	Optional[int]							=      AutoConfig.from_pretrained("""bert-base-cased"""      )
       lowerCamelCase		:	Optional[Any]							=      ["""config.4.0.0.json"""]

       with tempfile.TemporaryDirectory() as tmp_dir:
        configuration.save_pretrained(__magic_name__      )
        lowerCamelCase		:	str							=      2
        json.dump(configuration.to_dict()    ,  open(os.path.join(__magic_name__    ,  """config.4.0.0.json"""      )    ,  """w"""      )      )

        # This should pick the new configuration file as the version of Transformers is > 4.0.0
        lowerCamelCase		:	Union[str, Any]							=      AutoConfig.from_pretrained(__magic_name__      )
        self.assertEqual(new_configuration.hidden_size    ,  2      )

        # Will need to be adjusted if we reach v42 and this test is still here.
        # Should pick the old configuration file as the version of Transformers is < 4.42.0
        lowerCamelCase		:	Any							=      ["""config.42.0.0.json"""]
        lowerCamelCase		:	Optional[Any]							=      7_6_8
        configuration.save_pretrained(__magic_name__      )
        shutil.move(os.path.join(__magic_name__    ,  """config.4.0.0.json"""      )    ,  os.path.join(__magic_name__    ,  """config.42.0.0.json"""      )      )
        lowerCamelCase		:	int							=      AutoConfig.from_pretrained(__magic_name__      )
        self.assertEqual(new_configuration.hidden_size    ,  7_6_8      )






      def 						UpperCamelCase__							(      self      ):
       # This repo has two configuration files, one for v4.0.0 and above with a different hidden size.
       lowerCamelCase		:	str							=      """hf-internal-testing/test-two-configs"""

       import transformers as new_transformers

       lowerCamelCase		:	Tuple							=      """v4.0.0"""
       lowerCamelCase			,    lowerCamelCase		:	Optional[int]							=      new_transformers.models.auto.AutoConfig.from_pretrained(
           __magic_name__    ,  return_unused_kwargs=__magic_name__      )
       self.assertEqual(new_configuration.hidden_size    ,  2      )
       # This checks `_configuration_file` ia not kept in the kwargs by mistake.
       self.assertDictEqual(__magic_name__    ,  {}      )

       # Testing an older version by monkey-patching the version in the module it's used.
       import transformers as old_transformers

       lowerCamelCase		:	Tuple							=      """v3.0.0"""
       lowerCamelCase		:	Any							=      old_transformers.models.auto.AutoConfig.from_pretrained(__magic_name__      )
       self.assertEqual(old_configuration.hidden_size    ,  7_6_8      )




