


import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
   from test_module.custom_tokenization_fast import CustomTokenizerFast
class 	UpperCAmelCase_					(					unittest.TestCase    ):

   '''simple docstring'''




   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					List[Any]    =				0






   @slow
   def _snake_case    (		self ):






     """simple docstring"""





     for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
       lowerCamelCase :					Any    =				AutoTokenizer.from_pretrained(__A )
       self.assertIsNotNone(__A )
       self.assertIsInstance(__A						,				(BertTokenizer, BertTokenizerFast) )
       self.assertGreater(len(__A )						,				0 )

     for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
       lowerCamelCase :					Dict    =				AutoTokenizer.from_pretrained(__A )
       self.assertIsNotNone(__A )
       self.assertIsInstance(__A						,				(GPTaTokenizer, GPTaTokenizerFast) )
       self.assertGreater(len(__A )						,				0 )






   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					List[Any]    =				AutoTokenizer.from_pretrained(__A )
     self.assertIsInstance(__A						,				(BertTokenizer, BertTokenizerFast) )
     self.assertEqual(tokenizer.vocab_size						,				12 )






   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					int    =				AutoTokenizer.from_pretrained(__A )
     self.assertIsInstance(__A						,				(RobertaTokenizer, RobertaTokenizerFast) )
     self.assertEqual(tokenizer.vocab_size						,				20 )






   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					List[Any]    =				AutoConfig.from_pretrained(__A )
     self.assertIsInstance(__A						,				__A )
     # Check that tokenizer_type ≠ model_type
     lowerCamelCase :					int    =				AutoTokenizer.from_pretrained(__A						,				config=__A )
     self.assertIsInstance(__A						,				(BertTokenizer, BertTokenizerFast) )
     self.assertEqual(tokenizer.vocab_size						,				12 )






   def _snake_case    (		self ):






     """simple docstring"""





     with tempfile.TemporaryDirectory() as tmp_dir:
       shutil.copy("./tests/fixtures/vocab.txt"						,				os.path.join(__A						,				"vocab.txt" ) )

       lowerCamelCase :					Any    =				AutoTokenizer.from_pretrained(__A						,				tokenizer_type="bert"						,				use_fast=__A )
       self.assertIsInstance(__A						,				__A )

     with tempfile.TemporaryDirectory() as tmp_dir:
       shutil.copy("./tests/fixtures/vocab.json"						,				os.path.join(__A						,				"vocab.json" ) )
       shutil.copy("./tests/fixtures/merges.txt"						,				os.path.join(__A						,				"merges.txt" ) )

       lowerCamelCase :					Optional[int]    =				AutoTokenizer.from_pretrained(__A						,				tokenizer_type="gpt2"						,				use_fast=__A )
       self.assertIsInstance(__A						,				__A )






   @require_tokenizers
   def _snake_case    (		self ):






     """simple docstring"""





     with tempfile.TemporaryDirectory() as tmp_dir:
       shutil.copy("./tests/fixtures/vocab.txt"						,				os.path.join(__A						,				"vocab.txt" ) )

       lowerCamelCase :					int    =				AutoTokenizer.from_pretrained(__A						,				tokenizer_type="bert" )
       self.assertIsInstance(__A						,				__A )

     with tempfile.TemporaryDirectory() as tmp_dir:
       shutil.copy("./tests/fixtures/vocab.json"						,				os.path.join(__A						,				"vocab.json" ) )
       shutil.copy("./tests/fixtures/merges.txt"						,				os.path.join(__A						,				"merges.txt" ) )

       lowerCamelCase :					str    =				AutoTokenizer.from_pretrained(__A						,				tokenizer_type="gpt2" )
       self.assertIsInstance(__A						,				__A )






   def _snake_case    (		self ):






     """simple docstring"""





     with pytest.raises(__A ):
       AutoTokenizer.from_pretrained("./"						,				tokenizer_type="xxx" )






   @require_tokenizers
   def _snake_case    (		self ):






     """simple docstring"""





     for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
       lowerCamelCase :					Union[str, Any]    =				tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased" )
       self.assertIsInstance(__A						,				(BertTokenizer, BertTokenizerFast) )

       if isinstance(__A						,				__A ):
         self.assertEqual(tokenizer.basic_tokenizer.do_lower_case						,				__A )
       else:
         self.assertEqual(tokenizer.do_lower_case						,				__A )

       self.assertEqual(tokenizer.model_max_length						,				512 )






   @require_tokenizers
   def _snake_case    (		self ):






     """simple docstring"""





     for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
       with self.assertRaisesRegex(
           __A						,				"julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"						,				):
         lowerCamelCase :					Tuple    =				tokenizer_class.from_pretrained("julien-c/herlolip-not-exists" )






   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					List[str]    =				TOKENIZER_MAPPING.values()
     lowerCamelCase :					int    =				[]

     for slow_tok, fast_tok in tokenizers:
       if slow_tok is not None:
         tokenizer_names.append(slow_tok.__name__ )

       if fast_tok is not None:
         tokenizer_names.append(fast_tok.__name__ )

     for tokenizer_name in tokenizer_names:
       # must find the right class
       tokenizer_class_from_name(__A )






   @require_tokenizers
   def _snake_case    (		self ):






     """simple docstring"""





     self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"						,				use_fast=__A )						,				__A )
     self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased" )						,				__A )






   @require_tokenizers
   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					Dict    =				AutoTokenizer.from_pretrained("distilbert-base-uncased"						,				do_lower_case=__A )
     lowerCamelCase :					List[str]    =				"Hello, world. How are you?"
     lowerCamelCase :					Union[str, Any]    =				tokenizer.tokenize(__A )
     self.assertEqual("[UNK]"						,				tokens[0] )

     lowerCamelCase :					List[str]    =				AutoTokenizer.from_pretrained("microsoft/mpnet-base"						,				do_lower_case=__A )
     lowerCamelCase :					Tuple    =				tokenizer.tokenize(__A )
     self.assertEqual("[UNK]"						,				tokens[0] )






   @require_tokenizers
   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					Dict    =				AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config" )
     self.assertEqual(type(__A )						,				__A )
     self.assertEqual(tokenizer.model_max_length						,				512 )
     self.assertEqual(tokenizer.vocab_size						,				3_0000 )
     self.assertEqual(tokenizer.unk_token						,				"[UNK]" )
     self.assertEqual(tokenizer.padding_side						,				"right" )
     self.assertEqual(tokenizer.truncation_side						,				"right" )






   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					Dict    =				AutoTokenizer.from_pretrained(__A )
     self.assertIsInstance(__A						,				(BertTokenizer, BertTokenizerFast) )
     with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(__A )
       lowerCamelCase :					int    =				AutoTokenizer.from_pretrained(__A )

     self.assertIsInstance(__A						,				tokenizer.__class__ )
     self.assertEqual(tokenizera.vocab_size						,				12 )






   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					Any    =				AutoTokenizer.from_pretrained("ctrl" )
     # There is no fast CTRL so this always gives us a slow tokenizer.
     self.assertIsInstance(__A						,				__A )






   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					Tuple    =				get_tokenizer_config("bert-base-cased" )
     lowerCamelCase :					Optional[Any]    =				config.pop("_commit_hash"						,				__A )
     # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
     self.assertEqual(__A						,				{"do_lower_case": False} )

     # This model does not have a tokenizer_config so we get back an empty dict.
     lowerCamelCase :					List[str]    =				get_tokenizer_config(__A )
     self.assertDictEqual(__A						,				{} )

     # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
     lowerCamelCase :					Dict    =				AutoTokenizer.from_pretrained(__A )
     with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(__A )
       lowerCamelCase :					List[str]    =				get_tokenizer_config(__A )

     # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
     self.assertEqual(config["tokenizer_class"]						,				"BertTokenizer" )






   def _snake_case    (		self ):






     """simple docstring"""





     try:
       AutoConfig.register("custom"						,				__A )

       AutoTokenizer.register(__A						,				slow_tokenizer_class=__A )
       # Trying to register something existing in the Transformers library will raise an error
       with self.assertRaises(__A ):
         AutoTokenizer.register(__A						,				slow_tokenizer_class=__A )

       lowerCamelCase :					Any    =				CustomTokenizer.from_pretrained(__A )
       with tempfile.TemporaryDirectory() as tmp_dir:
         tokenizer.save_pretrained(__A )

         lowerCamelCase :					Optional[Any]    =				AutoTokenizer.from_pretrained(__A )
         self.assertIsInstance(__A						,				__A )

     finally:
       if "custom" in CONFIG_MAPPING._extra_content:
         del CONFIG_MAPPING._extra_content["custom"]
       if CustomConfig in TOKENIZER_MAPPING._extra_content:
         del TOKENIZER_MAPPING._extra_content[CustomConfig]






   @require_tokenizers
   def _snake_case    (		self ):






     """simple docstring"""





     try:
       AutoConfig.register("custom"						,				__A )

       # Can register in two steps
       AutoTokenizer.register(__A						,				slow_tokenizer_class=__A )
       self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,				(CustomTokenizer, None) )
       AutoTokenizer.register(__A						,				fast_tokenizer_class=__A )
       self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,				(CustomTokenizer, CustomTokenizerFast) )

       del TOKENIZER_MAPPING._extra_content[CustomConfig]
       # Can register in one step
       AutoTokenizer.register(
           __A						,				slow_tokenizer_class=__A						,				fast_tokenizer_class=__A )
       self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,				(CustomTokenizer, CustomTokenizerFast) )

       # Trying to register something existing in the Transformers library will raise an error
       with self.assertRaises(__A ):
         AutoTokenizer.register(__A						,				fast_tokenizer_class=__A )

       # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
       # and that model does not have a tokenizer.json
       with tempfile.TemporaryDirectory() as tmp_dir:
         lowerCamelCase :					Any    =				BertTokenizerFast.from_pretrained(__A )
         bert_tokenizer.save_pretrained(__A )
         lowerCamelCase :					Union[str, Any]    =				CustomTokenizerFast.from_pretrained(__A )

       with tempfile.TemporaryDirectory() as tmp_dir:
         tokenizer.save_pretrained(__A )

         lowerCamelCase :					Union[str, Any]    =				AutoTokenizer.from_pretrained(__A )
         self.assertIsInstance(__A						,				__A )

         lowerCamelCase :					int    =				AutoTokenizer.from_pretrained(__A						,				use_fast=__A )
         self.assertIsInstance(__A						,				__A )

     finally:
       if "custom" in CONFIG_MAPPING._extra_content:
         del CONFIG_MAPPING._extra_content["custom"]
       if CustomConfig in TOKENIZER_MAPPING._extra_content:
         del TOKENIZER_MAPPING._extra_content[CustomConfig]






   def _snake_case    (		self ):






     """simple docstring"""





     with self.assertRaises(__A ):
       lowerCamelCase :					str    =				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer" )
     # If remote code is disabled, we can't load this config.
     with self.assertRaises(__A ):
       lowerCamelCase :					Optional[Any]    =				AutoTokenizer.from_pretrained(
           "hf-internal-testing/test_dynamic_tokenizer"						,				trust_remote_code=__A )

     lowerCamelCase :					Any    =				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						,				trust_remote_code=__A )
     self.assertTrue(tokenizer.special_attribute_present )
     # Test tokenizer can be reloaded.
     with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(__A )
       lowerCamelCase :					int    =				AutoTokenizer.from_pretrained(__A						,				trust_remote_code=__A )
     self.assertTrue(reloaded_tokenizer.special_attribute_present )

     if is_tokenizers_available():
       self.assertEqual(tokenizer.__class__.__name__						,				"NewTokenizerFast" )
       self.assertEqual(reloaded_tokenizer.__class__.__name__						,				"NewTokenizerFast" )

       # Test we can also load the slow version
       lowerCamelCase :					List[Any]    =				AutoTokenizer.from_pretrained(
           "hf-internal-testing/test_dynamic_tokenizer"						,				trust_remote_code=__A						,				use_fast=__A )
       self.assertTrue(tokenizer.special_attribute_present )
       self.assertEqual(tokenizer.__class__.__name__						,				"NewTokenizer" )
       # Test tokenizer can be reloaded.
       with tempfile.TemporaryDirectory() as tmp_dir:
         tokenizer.save_pretrained(__A )
         lowerCamelCase :					Dict    =				AutoTokenizer.from_pretrained(__A						,				trust_remote_code=__A						,				use_fast=__A )
       self.assertEqual(reloaded_tokenizer.__class__.__name__						,				"NewTokenizer" )
       self.assertTrue(reloaded_tokenizer.special_attribute_present )
     else:
       self.assertEqual(tokenizer.__class__.__name__						,				"NewTokenizer" )
       self.assertEqual(reloaded_tokenizer.__class__.__name__						,				"NewTokenizer" )






   @require_tokenizers
   def _snake_case    (		self ):






     """simple docstring"""





     class 	UpperCAmelCase_					(					UpperCamelCase    ):

        '''simple docstring'''




        __A :      str     		=		False



     class 	UpperCAmelCase_					(					UpperCamelCase    ):

        '''simple docstring'''




        __A :      str     		=		NewTokenizer
        __A :      str     		=		False

     try:
       AutoConfig.register("custom"						,				__A )
       AutoTokenizer.register(__A						,				slow_tokenizer_class=__A )
       AutoTokenizer.register(__A						,				fast_tokenizer_class=__A )
       # If remote code is not set, the default is to use local
       lowerCamelCase :					Dict    =				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer" )
       self.assertEqual(tokenizer.__class__.__name__						,				"NewTokenizerFast" )
       self.assertFalse(tokenizer.special_attribute_present )
       lowerCamelCase :					Dict    =				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						,				use_fast=__A )
       self.assertEqual(tokenizer.__class__.__name__						,				"NewTokenizer" )
       self.assertFalse(tokenizer.special_attribute_present )

       # If remote code is disabled, we load the local one.
       lowerCamelCase :					Union[str, Any]    =				AutoTokenizer.from_pretrained(
           "hf-internal-testing/test_dynamic_tokenizer"						,				trust_remote_code=__A )
       self.assertEqual(tokenizer.__class__.__name__						,				"NewTokenizerFast" )
       self.assertFalse(tokenizer.special_attribute_present )
       lowerCamelCase :					Tuple    =				AutoTokenizer.from_pretrained(
           "hf-internal-testing/test_dynamic_tokenizer"						,				trust_remote_code=__A						,				use_fast=__A )
       self.assertEqual(tokenizer.__class__.__name__						,				"NewTokenizer" )
       self.assertFalse(tokenizer.special_attribute_present )

       # If remote is enabled, we load from the Hub
       lowerCamelCase :					Any    =				AutoTokenizer.from_pretrained(
           "hf-internal-testing/test_dynamic_tokenizer"						,				trust_remote_code=__A )
       self.assertEqual(tokenizer.__class__.__name__						,				"NewTokenizerFast" )
       self.assertTrue(tokenizer.special_attribute_present )
       lowerCamelCase :					Any    =				AutoTokenizer.from_pretrained(
           "hf-internal-testing/test_dynamic_tokenizer"						,				trust_remote_code=__A						,				use_fast=__A )
       self.assertEqual(tokenizer.__class__.__name__						,				"NewTokenizer" )
       self.assertTrue(tokenizer.special_attribute_present )

     finally:
       if "custom" in CONFIG_MAPPING._extra_content:
         del CONFIG_MAPPING._extra_content["custom"]
       if CustomConfig in TOKENIZER_MAPPING._extra_content:
         del TOKENIZER_MAPPING._extra_content[CustomConfig]






   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					Any    =				AutoTokenizer.from_pretrained(
         "hf-internal-testing/test_dynamic_tokenizer_legacy"						,				trust_remote_code=__A )
     self.assertTrue(tokenizer.special_attribute_present )
     if is_tokenizers_available():
       self.assertEqual(tokenizer.__class__.__name__						,				"NewTokenizerFast" )

       # Test we can also load the slow version
       lowerCamelCase :					Tuple    =				AutoTokenizer.from_pretrained(
           "hf-internal-testing/test_dynamic_tokenizer_legacy"						,				trust_remote_code=__A						,				use_fast=__A )
       self.assertTrue(tokenizer.special_attribute_present )
       self.assertEqual(tokenizer.__class__.__name__						,				"NewTokenizer" )
     else:
       self.assertEqual(tokenizer.__class__.__name__						,				"NewTokenizer" )






   def _snake_case    (		self ):






     """simple docstring"""





     with self.assertRaisesRegex(
         __A						,				"bert-base is not a local folder and is not a valid model identifier" ):
       lowerCamelCase :					Any    =				AutoTokenizer.from_pretrained("bert-base" )






   def _snake_case    (		self ):






     """simple docstring"""





     with self.assertRaisesRegex(
         __A						,				R"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)" ):
       lowerCamelCase :					Dict    =				AutoTokenizer.from_pretrained(__A						,				revision="aaaaaa" )






   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					Optional[int]    =				AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert" )
     with RequestCounter() as counter:
       lowerCamelCase :					Dict    =				AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert" )
       self.assertEqual(counter.get_request_count						,				0 )
       self.assertEqual(counter.head_request_count						,				1 )
       self.assertEqual(counter.other_request_count						,				0 )







from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType
class 	UpperCAmelCase_					(					UpperCamelCase    ):

   '''simple docstring'''




   __A :      Dict     		=		["image_processor", "tokenizer"]
   __A :      Dict     		=		"BridgeTowerImageProcessor"
   __A :      Optional[int]     		=		("RobertaTokenizer", "RobertaTokenizerFast")






   def __init__(		self						,				__A						,				__A ):






     """simple docstring"""





     super().__init__(__A						,				__A )






   def __call__(		self						,				__A						,				__A = None						,				__A = True						,				__A = False						,				__A = None						,				__A = None						,				__A = 0						,				__A = None						,				__A = None						,				__A = None						,				__A = False						,				__A = False						,				__A = False						,				__A = False						,				__A = True						,				__A = None						,				**__A						,				):






     """simple docstring"""





     lowerCamelCase :					str    =				self.tokenizer(
         text=__A						,				add_special_tokens=__A						,				padding=__A						,				truncation=__A						,				max_length=__A						,				stride=__A						,				pad_to_multiple_of=__A						,				return_token_type_ids=__A						,				return_attention_mask=__A						,				return_overflowing_tokens=__A						,				return_special_tokens_mask=__A						,				return_offsets_mapping=__A						,				return_length=__A						,				verbose=__A						,				return_tensors=__A						,				**__A						,				)
     # add pixel_values + pixel_mask
     lowerCamelCase :					int    =				self.image_processor(
         __A						,				return_tensors=__A						,				do_normalize=__A						,				do_center_crop=__A						,				**__A )
     encoding.update(__A )

     return encoding






   def _snake_case    (		self						,				*__A						,				**__A ):






     """simple docstring"""





     return self.tokenizer.batch_decode(*__A						,				**__A )






   def _snake_case    (		self						,				*__A						,				**__A ):






     """simple docstring"""





     return self.tokenizer.decode(*__A						,				**__A )






   @property
   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					List[Any]    =				self.tokenizer.model_input_names
     lowerCamelCase :					int    =				self.image_processor.model_input_names
     return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names ) )



