





import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		UpperCamelCase__ (      lowerCAmelCase_							,      unittest.TestCase   ):



     '''simple docstring'''

     __snake_case    :	str			=			OpenAIGPTTokenizer
     __snake_case    :	List[str]			=			OpenAIGPTTokenizerFast
     __snake_case    :	Tuple			=			True
     __snake_case    :	int			=			False






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Dict       )     ->		List[str]:




           '''simple docstring'''


           super().setUp()

           # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
           SCREAMING_SNAKE_CASE		=						[
               """l""",
               """o""",
               """w""",
               """e""",
               """r""",
               """s""",
               """t""",
               """i""",
               """d""",
               """n""",
               """w</w>""",
               """r</w>""",
               """t</w>""",
               """lo""",
               """low""",
               """er</w>""",
               """low</w>""",
               """lowest</w>""",
               """newer</w>""",
               """wider</w>""",
               """<unk>""",
           ]
           SCREAMING_SNAKE_CASE		=						dict(zip(lowerCamelCase__						,range(len(lowerCamelCase__       )       )       )       )
           SCREAMING_SNAKE_CASE		=						["""#version: 0.2""", """l o""", """lo w""", """e r</w>""", """"""]

           SCREAMING_SNAKE_CASE		=						os.path.join(self.tmpdirname						,VOCAB_FILES_NAMES["""vocab_file"""]       )
           SCREAMING_SNAKE_CASE		=						os.path.join(self.tmpdirname						,VOCAB_FILES_NAMES["""merges_file"""]       )
           with open(self.vocab_file						,"""w"""       ) as fp:
                 fp.write(json.dumps(lowerCamelCase__       )       )
           with open(self.merges_file						,"""w"""       ) as fp:
                 fp.write("""\n""".join(lowerCamelCase__       )       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Tuple						,lowerCamelCase__						:    Dict       )     ->		Optional[int]:




           '''simple docstring'''


           return "lower newer", "lower newer"






     def 				SCREAMING_SNAKE_CASE__  (							self						:    List[str]       )     ->		Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						OpenAIGPTTokenizer(self.vocab_file						,self.merges_file       )

           SCREAMING_SNAKE_CASE		=						"""lower"""
           SCREAMING_SNAKE_CASE		=						["""low""", """er</w>"""]
           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize(lowerCamelCase__       )
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__       )

           SCREAMING_SNAKE_CASE		=						tokens + ["""<unk>"""]
           SCREAMING_SNAKE_CASE		=						[14, 15, 20]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__       )						,lowerCamelCase__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Dict						,lowerCamelCase__						:    List[Any]=15       )     ->		Tuple:




           '''simple docstring'''


           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
                       SCREAMING_SNAKE_CASE		=						self.rust_tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__       )

                       # Simple input
                       SCREAMING_SNAKE_CASE		=						"""This is a simple input"""
                       SCREAMING_SNAKE_CASE		=						["""This is a simple input 1""", """This is a simple input 2"""]
                       SCREAMING_SNAKE_CASE		=						("""This is a simple input""", """This is a pair""")
                       SCREAMING_SNAKE_CASE		=						[
                           ("""This is a simple input 1""", """This is a simple input 2"""),
                           ("""This is a simple pair 1""", """This is a simple pair 2"""),
                       ]

                       # Simple input tests
                       self.assertRaises(lowerCamelCase__						,tokenizer_r.encode						,lowerCamelCase__						,max_length=lowerCamelCase__						,padding="""max_length"""       )

                       # Simple input
                       self.assertRaises(lowerCamelCase__						,tokenizer_r.encode_plus						,lowerCamelCase__						,max_length=lowerCamelCase__						,padding="""max_length"""       )

                       # Simple input
                       self.assertRaises(
                           lowerCamelCase__						,tokenizer_r.batch_encode_plus						,lowerCamelCase__						,max_length=lowerCamelCase__						,padding="""max_length"""						,)

                       # Pair input
                       self.assertRaises(lowerCamelCase__						,tokenizer_r.encode						,lowerCamelCase__						,max_length=lowerCamelCase__						,padding="""max_length"""       )

                       # Pair input
                       self.assertRaises(lowerCamelCase__						,tokenizer_r.encode_plus						,lowerCamelCase__						,max_length=lowerCamelCase__						,padding="""max_length"""       )

                       # Pair input
                       self.assertRaises(
                           lowerCamelCase__						,tokenizer_r.batch_encode_plus						,lowerCamelCase__						,max_length=lowerCamelCase__						,padding="""max_length"""						,)






     def 				SCREAMING_SNAKE_CASE__  (							self						:    str       )     ->		List[str]:




           '''simple docstring'''


           pass

@require_ftfy
@require_spacy
@require_tokenizers
class 		UpperCamelCase__ (      lowerCAmelCase_   ):



     '''simple docstring'''

     pass









import importlib
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
import transformers.models.auto
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.bert.configuration_bert import BertConfig
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402


SCREAMING_SNAKE_CASE_						   =       get_tests_dir("""fixtures/dummy-config.json""")

class 		UpperCamelCase__ (      unittest.TestCase   ):



     '''simple docstring'''

     def 				SCREAMING_SNAKE_CASE__  (							self						:    Optional[Any]       )     ->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						0






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Any       )     ->		str:




           '''simple docstring'''


           self.assertIsNotNone(transformers.models.auto.__spec__       )
           self.assertIsNotNone(importlib.util.find_spec("""transformers.models.auto"""       )       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Union[str, Any]       )     ->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained("""bert-base-uncased"""       )
           self.assertIsInstance(lowerCamelCase__						,lowerCamelCase__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    int       )     ->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained(lowerCamelCase__       )
           self.assertIsInstance(lowerCamelCase__						,lowerCamelCase__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Union[str, Any]       )     ->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained(lowerCamelCase__       )
           self.assertIsInstance(lowerCamelCase__						,lowerCamelCase__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Tuple       )     ->		Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						AutoConfig.for_model("""roberta"""       )
           self.assertIsInstance(lowerCamelCase__						,lowerCamelCase__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    int       )     ->		int:




           '''simple docstring'''


           with tempfile.TemporaryDirectory() as tmp_dir:
                 # This model name contains bert and roberta, but roberta ends up being picked.
                 SCREAMING_SNAKE_CASE		=						os.path.join(lowerCamelCase__						,"""fake-roberta"""       )
                 os.makedirs(lowerCamelCase__						,exist_ok=lowerCamelCase__       )
                 with open(os.path.join(lowerCamelCase__						,"""config.json"""       )						,"""w"""       ) as f:
                       f.write(json.dumps({}       )       )
                 SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained(lowerCamelCase__       )
                 self.assertEqual(type(lowerCamelCase__       )						,lowerCamelCase__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Dict       )     ->		str:




           '''simple docstring'''


           try:
                 AutoConfig.register("""custom"""						,lowerCamelCase__       )
                 # Wrong model type will raise an error
                 with self.assertRaises(lowerCamelCase__       ):
                       AutoConfig.register("""model"""						,lowerCamelCase__       )
                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(lowerCamelCase__       ):
                       AutoConfig.register("""bert"""						,lowerCamelCase__       )

                 # Now that the config is registered, it can be used as any other config with the auto-API
                 SCREAMING_SNAKE_CASE		=						CustomConfig()
                 with tempfile.TemporaryDirectory() as tmp_dir:
                       config.save_pretrained(lowerCamelCase__       )
                       SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained(lowerCamelCase__       )
                       self.assertIsInstance(lowerCamelCase__						,lowerCamelCase__       )

           finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["custom"]






     def 				SCREAMING_SNAKE_CASE__  (							self						:    str       )     ->		Dict:




           '''simple docstring'''


           with self.assertRaisesRegex(
               lowerCamelCase__						,"""bert-base is not a local folder and is not a valid model identifier"""       ):
                 SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained("""bert-base"""       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Dict       )     ->		str:




           '''simple docstring'''


           with self.assertRaisesRegex(
               lowerCamelCase__						,R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""       ):
                 SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained(lowerCamelCase__						,revision="""aaaaaa"""       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Tuple       )     ->		List[Any]:




           '''simple docstring'''


           with self.assertRaisesRegex(
               lowerCamelCase__						,"""hf-internal-testing/no-config-test-repo does not appear to have a file named config.json."""						,):
                 SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained("""hf-internal-testing/no-config-test-repo"""       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Optional[int]       )     ->		Union[str, Any]:




           '''simple docstring'''


           with self.assertRaises(lowerCamelCase__       ):
                 SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""       )
           # If remote code is disabled, we can't load this config.
           with self.assertRaises(lowerCamelCase__       ):
                 SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""						,trust_remote_code=lowerCamelCase__       )

           SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""						,trust_remote_code=lowerCamelCase__       )
           self.assertEqual(config.__class__.__name__						,"""NewModelConfig"""       )

           # Test config can be reloaded.
           with tempfile.TemporaryDirectory() as tmp_dir:
                 config.save_pretrained(lowerCamelCase__       )
                 SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained(lowerCamelCase__						,trust_remote_code=lowerCamelCase__       )
           self.assertEqual(reloaded_config.__class__.__name__						,"""NewModelConfig"""       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Dict       )     ->		Union[str, Any]:




           '''simple docstring'''


           class 		UpperCamelCase__ (      lowerCAmelCase_   ):



                '''simple docstring'''

                __snake_case    :	Union[str, Any]			=			"new-model"

           try:
                 AutoConfig.register("""new-model"""						,lowerCamelCase__       )
                 # If remote code is not set, the default is to use local
                 SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""       )
                 self.assertEqual(config.__class__.__name__						,"""NewModelConfigLocal"""       )

                 # If remote code is disabled, we load the local one.
                 SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""						,trust_remote_code=lowerCamelCase__       )
                 self.assertEqual(config.__class__.__name__						,"""NewModelConfigLocal"""       )

                 # If remote is enabled, we load from the Hub
                 SCREAMING_SNAKE_CASE		=						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""						,trust_remote_code=lowerCamelCase__       )
                 self.assertEqual(config.__class__.__name__						,"""NewModelConfig"""       )

           finally:
                 if "new-model" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["new-model"]


