

'''simple docstring'''



import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast







class 						lowerCAmelCase__   (      unittest.TestCase					):



 """simple docstring"""



 def       UpperCAmelCase__		(		self					:       Dict			)  -> Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      0
 @slow
 def       UpperCAmelCase__		(		self					:       Tuple			)  -> Optional[Any]:


      """simple docstring"""




      for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			)
           self.assertIsNotNone(__SCREAMING_SNAKE_CASE			)
           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							(BertTokenizer, BertTokenizerFast)			)
           self.assertGreater(len(__SCREAMING_SNAKE_CASE			)					,							0			)

      for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			)
           self.assertIsNotNone(__SCREAMING_SNAKE_CASE			)
           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							(GPTaTokenizer, GPTaTokenizerFast)			)
           self.assertGreater(len(__SCREAMING_SNAKE_CASE			)					,							0			)
 def       UpperCAmelCase__		(		self					:       str			)  -> Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			)
      self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							(BertTokenizer, BertTokenizerFast)			)
      self.assertEqual(tokenizer.vocab_size					,							12			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			)
      self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							(RobertaTokenizer, RobertaTokenizerFast)			)
      self.assertEqual(tokenizer.vocab_size					,							20			)
 def       UpperCAmelCase__		(		self					:       Any			)  -> str:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      AutoConfig.from_pretrained(__SCREAMING_SNAKE_CASE			)
      self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
      # Check that tokenizer_type ≠ model_type
      __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,							config=__SCREAMING_SNAKE_CASE			)
      self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							(BertTokenizer, BertTokenizerFast)			)
      self.assertEqual(tokenizer.vocab_size					,							12			)
 def       UpperCAmelCase__		(		self					:       str			)  -> Any:


      """simple docstring"""




      with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("""./tests/fixtures/vocab.txt"""					,							os.path.join(__SCREAMING_SNAKE_CASE					,							"""vocab.txt"""			)			)

           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,							tokenizer_type="""bert"""					,							use_fast=__SCREAMING_SNAKE_CASE			)
           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)

      with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("""./tests/fixtures/vocab.json"""					,							os.path.join(__SCREAMING_SNAKE_CASE					,							"""vocab.json"""			)			)
           shutil.copy("""./tests/fixtures/merges.txt"""					,							os.path.join(__SCREAMING_SNAKE_CASE					,							"""merges.txt"""			)			)

           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,							tokenizer_type="""gpt2"""					,							use_fast=__SCREAMING_SNAKE_CASE			)
           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
 @require_tokenizers
 def       UpperCAmelCase__		(		self					:       Tuple			)  -> List[Any]:


      """simple docstring"""




      with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("""./tests/fixtures/vocab.txt"""					,							os.path.join(__SCREAMING_SNAKE_CASE					,							"""vocab.txt"""			)			)

           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,							tokenizer_type="""bert"""			)
           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)

      with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("""./tests/fixtures/vocab.json"""					,							os.path.join(__SCREAMING_SNAKE_CASE					,							"""vocab.json"""			)			)
           shutil.copy("""./tests/fixtures/merges.txt"""					,							os.path.join(__SCREAMING_SNAKE_CASE					,							"""merges.txt"""			)			)

           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,							tokenizer_type="""gpt2"""			)
           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       List[str]			)  -> Union[str, Any]:


      """simple docstring"""




      with pytest.raises(__SCREAMING_SNAKE_CASE			):
           AutoTokenizer.from_pretrained("""./"""					,							tokenizer_type="""xxx"""			)
 @require_tokenizers
 def       UpperCAmelCase__		(		self					:       List[Any]			)  -> Tuple:


      """simple docstring"""




      for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
           __SCREAMING_SNAKE_CASE     =      tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""			)
           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							(BertTokenizer, BertTokenizerFast)			)

           if isinstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			):
                self.assertEqual(tokenizer.basic_tokenizer.do_lower_case					,							__SCREAMING_SNAKE_CASE			)
           else:
                self.assertEqual(tokenizer.do_lower_case					,							__SCREAMING_SNAKE_CASE			)

           self.assertEqual(tokenizer.model_max_length					,							512			)
 @require_tokenizers
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> Any:


      """simple docstring"""




      for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
           with self.assertRaisesRegex(
               __SCREAMING_SNAKE_CASE					,							"""julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""					,							):
                __SCREAMING_SNAKE_CASE     =      tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""			)
 def       UpperCAmelCase__		(		self					:       List[Any]			)  -> Dict:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      TOKENIZER_MAPPING.values()
      __SCREAMING_SNAKE_CASE     =      []

      for slow_tok, fast_tok in tokenizers:
           if slow_tok is not None:
                tokenizer_names.append(slow_tok.__name__			)

           if fast_tok is not None:
                tokenizer_names.append(fast_tok.__name__			)

      for tokenizer_name in tokenizer_names:
           # must find the right class
           tokenizer_class_from_name(__SCREAMING_SNAKE_CASE			)
 @require_tokenizers
 def       UpperCAmelCase__		(		self					:       Dict			)  -> List[str]:


      """simple docstring"""




      self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""					,							use_fast=__SCREAMING_SNAKE_CASE			)					,							__SCREAMING_SNAKE_CASE			)
      self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""			)					,							__SCREAMING_SNAKE_CASE			)
 @require_tokenizers
 def       UpperCAmelCase__		(		self					:       Tuple			)  -> Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained("""distilbert-base-uncased"""					,							do_lower_case=__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      """Hello, world. How are you?"""
      __SCREAMING_SNAKE_CASE     =      tokenizer.tokenize(__SCREAMING_SNAKE_CASE			)
      self.assertEqual("""[UNK]"""					,							tokens[0]			)

      __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""					,							do_lower_case=__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      tokenizer.tokenize(__SCREAMING_SNAKE_CASE			)
      self.assertEqual("""[UNK]"""					,							tokens[0]			)
 @require_tokenizers
 def       UpperCAmelCase__		(		self					:       Any			)  -> Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""			)
      self.assertEqual(type(__SCREAMING_SNAKE_CASE			)					,							__SCREAMING_SNAKE_CASE			)
      self.assertEqual(tokenizer.model_max_length					,							512			)
      self.assertEqual(tokenizer.vocab_size					,							30_000			)
      self.assertEqual(tokenizer.unk_token					,							"""[UNK]"""			)
      self.assertEqual(tokenizer.padding_side					,							"""right"""			)
      self.assertEqual(tokenizer.truncation_side					,							"""right"""			)
 def       UpperCAmelCase__		(		self					:       int			)  -> Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			)
      self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							(BertTokenizer, BertTokenizerFast)			)
      with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE			)
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			)

      self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							tokenizer.__class__			)
      self.assertEqual(tokenizera.vocab_size					,							12			)
 def       UpperCAmelCase__		(		self					:       Union[str, Any]			)  -> int:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained("""ctrl"""			)
      # There is no fast CTRL so this always gives us a slow tokenizer.
      self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       Any			)  -> Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      get_tokenizer_config("""bert-base-cased"""			)
      __SCREAMING_SNAKE_CASE     =      config.pop("""_commit_hash"""					,							__SCREAMING_SNAKE_CASE			)
      # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
      self.assertEqual(__SCREAMING_SNAKE_CASE					,							{"""do_lower_case""": False}			)

      # This model does not have a tokenizer_config so we get back an empty dict.
      __SCREAMING_SNAKE_CASE     =      get_tokenizer_config(__SCREAMING_SNAKE_CASE			)
      self.assertDictEqual(__SCREAMING_SNAKE_CASE					,							{}			)

      # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
      __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			)
      with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE			)
           __SCREAMING_SNAKE_CASE     =      get_tokenizer_config(__SCREAMING_SNAKE_CASE			)

      # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
      self.assertEqual(config["""tokenizer_class"""]					,							"""BertTokenizer"""			)
 def       UpperCAmelCase__		(		self					:       Optional[int]			)  -> Optional[Any]:


      """simple docstring"""




      try:
           AutoConfig.register("""custom"""					,							__SCREAMING_SNAKE_CASE			)

           AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,							slow_tokenizer_class=__SCREAMING_SNAKE_CASE			)
           # Trying to register something existing in the Transformers library will raise an error
           with self.assertRaises(__SCREAMING_SNAKE_CASE			):
                AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,							slow_tokenizer_class=__SCREAMING_SNAKE_CASE			)

           __SCREAMING_SNAKE_CASE     =      CustomTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			)
           with tempfile.TemporaryDirectory() as tmp_dir:
                tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE			)

                __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			)
                self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)

      finally:
           if "custom" in CONFIG_MAPPING._extra_content:
                del CONFIG_MAPPING._extra_content["custom"]
           if CustomConfig in TOKENIZER_MAPPING._extra_content:
                del TOKENIZER_MAPPING._extra_content[CustomConfig]
 @require_tokenizers
 def       UpperCAmelCase__		(		self					:       Optional[int]			)  -> Tuple:


      """simple docstring"""




      try:
           AutoConfig.register("""custom"""					,							__SCREAMING_SNAKE_CASE			)

           # Can register in two steps
           AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,							slow_tokenizer_class=__SCREAMING_SNAKE_CASE			)
           self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,							(CustomTokenizer, None)			)
           AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,							fast_tokenizer_class=__SCREAMING_SNAKE_CASE			)
           self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,							(CustomTokenizer, CustomTokenizerFast)			)

           del TOKENIZER_MAPPING._extra_content[CustomConfig]
           # Can register in one step
           AutoTokenizer.register(
               __SCREAMING_SNAKE_CASE					,							slow_tokenizer_class=__SCREAMING_SNAKE_CASE					,							fast_tokenizer_class=__SCREAMING_SNAKE_CASE			)
           self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,							(CustomTokenizer, CustomTokenizerFast)			)

           # Trying to register something existing in the Transformers library will raise an error
           with self.assertRaises(__SCREAMING_SNAKE_CASE			):
                AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,							fast_tokenizer_class=__SCREAMING_SNAKE_CASE			)

           # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
           # and that model does not have a tokenizer.json
           with tempfile.TemporaryDirectory() as tmp_dir:
                __SCREAMING_SNAKE_CASE     =      BertTokenizerFast.from_pretrained(__SCREAMING_SNAKE_CASE			)
                bert_tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      CustomTokenizerFast.from_pretrained(__SCREAMING_SNAKE_CASE			)

           with tempfile.TemporaryDirectory() as tmp_dir:
                tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE			)

                __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			)
                self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)

                __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,							use_fast=__SCREAMING_SNAKE_CASE			)
                self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)

      finally:
           if "custom" in CONFIG_MAPPING._extra_content:
                del CONFIG_MAPPING._extra_content["custom"]
           if CustomConfig in TOKENIZER_MAPPING._extra_content:
                del TOKENIZER_MAPPING._extra_content[CustomConfig]
 def       UpperCAmelCase__		(		self					:       int			)  -> str:


      """simple docstring"""




      with self.assertRaises(__SCREAMING_SNAKE_CASE			):
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""			)
      # If remote code is disabled, we can't load this config.
      with self.assertRaises(__SCREAMING_SNAKE_CASE			):
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(
               """hf-internal-testing/test_dynamic_tokenizer"""					,							trust_remote_code=__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""					,							trust_remote_code=__SCREAMING_SNAKE_CASE			)
      self.assertTrue(tokenizer.special_attribute_present			)
      # Test tokenizer can be reloaded.
      with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE			)
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,							trust_remote_code=__SCREAMING_SNAKE_CASE			)
      self.assertTrue(reloaded_tokenizer.special_attribute_present			)

      if is_tokenizers_available():
           self.assertEqual(tokenizer.__class__.__name__					,							"""NewTokenizerFast"""			)
           self.assertEqual(reloaded_tokenizer.__class__.__name__					,							"""NewTokenizerFast"""			)

           # Test we can also load the slow version
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(
               """hf-internal-testing/test_dynamic_tokenizer"""					,							trust_remote_code=__SCREAMING_SNAKE_CASE					,							use_fast=__SCREAMING_SNAKE_CASE			)
           self.assertTrue(tokenizer.special_attribute_present			)
           self.assertEqual(tokenizer.__class__.__name__					,							"""NewTokenizer"""			)
           # Test tokenizer can be reloaded.
           with tempfile.TemporaryDirectory() as tmp_dir:
                tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,							trust_remote_code=__SCREAMING_SNAKE_CASE					,							use_fast=__SCREAMING_SNAKE_CASE			)
           self.assertEqual(reloaded_tokenizer.__class__.__name__					,							"""NewTokenizer"""			)
           self.assertTrue(reloaded_tokenizer.special_attribute_present			)
      else:
           self.assertEqual(tokenizer.__class__.__name__					,							"""NewTokenizer"""			)
           self.assertEqual(reloaded_tokenizer.__class__.__name__					,							"""NewTokenizer"""			)
 @require_tokenizers
 def       UpperCAmelCase__		(		self					:       Dict			)  -> Any:


      """simple docstring"""




      class 						lowerCAmelCase__   (      a					):



       """simple docstring"""



       lowerCAmelCase__  					=      False







      class 						lowerCAmelCase__   (      a					):



       """simple docstring"""



       lowerCAmelCase__  					=      NewTokenizer
       lowerCAmelCase__  					=      False

      try:
           AutoConfig.register("""custom"""					,							__SCREAMING_SNAKE_CASE			)
           AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,							slow_tokenizer_class=__SCREAMING_SNAKE_CASE			)
           AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,							fast_tokenizer_class=__SCREAMING_SNAKE_CASE			)
           # If remote code is not set, the default is to use local
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""			)
           self.assertEqual(tokenizer.__class__.__name__					,							"""NewTokenizerFast"""			)
           self.assertFalse(tokenizer.special_attribute_present			)
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""					,							use_fast=__SCREAMING_SNAKE_CASE			)
           self.assertEqual(tokenizer.__class__.__name__					,							"""NewTokenizer"""			)
           self.assertFalse(tokenizer.special_attribute_present			)

           # If remote code is disabled, we load the local one.
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(
               """hf-internal-testing/test_dynamic_tokenizer"""					,							trust_remote_code=__SCREAMING_SNAKE_CASE			)
           self.assertEqual(tokenizer.__class__.__name__					,							"""NewTokenizerFast"""			)
           self.assertFalse(tokenizer.special_attribute_present			)
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(
               """hf-internal-testing/test_dynamic_tokenizer"""					,							trust_remote_code=__SCREAMING_SNAKE_CASE					,							use_fast=__SCREAMING_SNAKE_CASE			)
           self.assertEqual(tokenizer.__class__.__name__					,							"""NewTokenizer"""			)
           self.assertFalse(tokenizer.special_attribute_present			)

           # If remote is enabled, we load from the Hub
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(
               """hf-internal-testing/test_dynamic_tokenizer"""					,							trust_remote_code=__SCREAMING_SNAKE_CASE			)
           self.assertEqual(tokenizer.__class__.__name__					,							"""NewTokenizerFast"""			)
           self.assertTrue(tokenizer.special_attribute_present			)
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(
               """hf-internal-testing/test_dynamic_tokenizer"""					,							trust_remote_code=__SCREAMING_SNAKE_CASE					,							use_fast=__SCREAMING_SNAKE_CASE			)
           self.assertEqual(tokenizer.__class__.__name__					,							"""NewTokenizer"""			)
           self.assertTrue(tokenizer.special_attribute_present			)

      finally:
           if "custom" in CONFIG_MAPPING._extra_content:
                del CONFIG_MAPPING._extra_content["custom"]
           if CustomConfig in TOKENIZER_MAPPING._extra_content:
                del TOKENIZER_MAPPING._extra_content[CustomConfig]
 def       UpperCAmelCase__		(		self					:       Optional[int]			)  -> Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(
          """hf-internal-testing/test_dynamic_tokenizer_legacy"""					,							trust_remote_code=__SCREAMING_SNAKE_CASE			)
      self.assertTrue(tokenizer.special_attribute_present			)
      if is_tokenizers_available():
           self.assertEqual(tokenizer.__class__.__name__					,							"""NewTokenizerFast"""			)

           # Test we can also load the slow version
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(
               """hf-internal-testing/test_dynamic_tokenizer_legacy"""					,							trust_remote_code=__SCREAMING_SNAKE_CASE					,							use_fast=__SCREAMING_SNAKE_CASE			)
           self.assertTrue(tokenizer.special_attribute_present			)
           self.assertEqual(tokenizer.__class__.__name__					,							"""NewTokenizer"""			)
      else:
           self.assertEqual(tokenizer.__class__.__name__					,							"""NewTokenizer"""			)
 def       UpperCAmelCase__		(		self					:       Tuple			)  -> Dict:


      """simple docstring"""




      with self.assertRaisesRegex(
          __SCREAMING_SNAKE_CASE					,							"""bert-base is not a local folder and is not a valid model identifier"""			):
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained("""bert-base"""			)
 def       UpperCAmelCase__		(		self					:       List[Any]			)  -> List[str]:


      """simple docstring"""




      with self.assertRaisesRegex(
          __SCREAMING_SNAKE_CASE					,							r"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""			):
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,							revision="""aaaaaa"""			)


 def       UpperCAmelCase__		(		self					:       Any			)  -> str:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""			)
      with RequestCounter() as counter:
           __SCREAMING_SNAKE_CASE     =      AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""			)
           self.assertEqual(counter.get_request_count					,							0			)
           self.assertEqual(counter.head_request_count					,							1			)
           self.assertEqual(counter.other_request_count					,							0			)









'''simple docstring'''



import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
  from PIL import Image

  from transformers import CLIPImageProcessor, CLIPProcessor







@require_vision
class 						lowerCAmelCase__   (      unittest.TestCase					):



 """simple docstring"""



 def       UpperCAmelCase__		(		self					:       Union[str, Any]			)  -> List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      tempfile.mkdtemp()

      # fmt: off
      __SCREAMING_SNAKE_CASE     =      ["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
      # fmt: on
      __SCREAMING_SNAKE_CASE     =      dict(zip(__SCREAMING_SNAKE_CASE					,							range(len(__SCREAMING_SNAKE_CASE			)			)			)			)
      __SCREAMING_SNAKE_CASE     =      ["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>""", """"""]
      __SCREAMING_SNAKE_CASE     =      {"""unk_token""": """<unk>"""}

      __SCREAMING_SNAKE_CASE     =      os.path.join(self.tmpdirname					,							VOCAB_FILES_NAMES["""vocab_file"""]			)
      __SCREAMING_SNAKE_CASE     =      os.path.join(self.tmpdirname					,							VOCAB_FILES_NAMES["""merges_file"""]			)
      with open(self.vocab_file					,							"""w"""					,							encoding="""utf-8"""			) as fp:
           fp.write(json.dumps(__SCREAMING_SNAKE_CASE			) + """\n"""			)
      with open(self.merges_file					,							"""w"""					,							encoding="""utf-8"""			) as fp:
           fp.write("""\n""".join(__SCREAMING_SNAKE_CASE			)			)

      __SCREAMING_SNAKE_CASE     =      {
          """do_resize""": True,
          """size""": 20,
          """do_center_crop""": True,
          """crop_size""": 18,
          """do_normalize""": True,
          """image_mean""": [0.48145466, 0.4578275, 0.40821073],
          """image_std""": [0.26862954, 0.26130258, 0.27577711],
      }
      __SCREAMING_SNAKE_CASE     =      os.path.join(self.tmpdirname					,							__SCREAMING_SNAKE_CASE			)
      with open(self.image_processor_file					,							"""w"""					,							encoding="""utf-8"""			) as fp:
           json.dump(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       Optional[int]					,							**__SCREAMING_SNAKE_CASE					:       Optional[int]			)  -> str:


      """simple docstring"""




      return CLIPTokenizer.from_pretrained(self.tmpdirname					,							**__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       Tuple					,							**__SCREAMING_SNAKE_CASE					:       Any			)  -> int:


      """simple docstring"""




      return CLIPTokenizerFast.from_pretrained(self.tmpdirname					,							**__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]					,							**__SCREAMING_SNAKE_CASE					:       Optional[int]			)  -> List[str]:


      """simple docstring"""




      return CLIPImageProcessor.from_pretrained(self.tmpdirname					,							**__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       Tuple			)  -> str:


      """simple docstring"""




      shutil.rmtree(self.tmpdirname			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> str:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      [np.random.randint(255					,							size=(3, 30, 400)					,							dtype=np.uinta			)]

      __SCREAMING_SNAKE_CASE     =      [Image.fromarray(np.moveaxis(__SCREAMING_SNAKE_CASE					,							0					,							-1			)			) for x in image_inputs]

      return image_inputs
 def       UpperCAmelCase__		(		self					:       Optional[int]			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.get_tokenizer()
      __SCREAMING_SNAKE_CASE     =      self.get_rust_tokenizer()
      __SCREAMING_SNAKE_CASE     =      self.get_image_processor()

      __SCREAMING_SNAKE_CASE     =      CLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,							image_processor=__SCREAMING_SNAKE_CASE			)
      processor_slow.save_pretrained(self.tmpdirname			)
      __SCREAMING_SNAKE_CASE     =      CLIPProcessor.from_pretrained(self.tmpdirname					,							use_fast=__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      CLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,							image_processor=__SCREAMING_SNAKE_CASE			)
      processor_fast.save_pretrained(self.tmpdirname			)
      __SCREAMING_SNAKE_CASE     =      CLIPProcessor.from_pretrained(self.tmpdirname			)

      self.assertEqual(processor_slow.tokenizer.get_vocab()					,							tokenizer_slow.get_vocab()			)
      self.assertEqual(processor_fast.tokenizer.get_vocab()					,							tokenizer_fast.get_vocab()			)
      self.assertEqual(tokenizer_slow.get_vocab()					,							tokenizer_fast.get_vocab()			)
      self.assertIsInstance(processor_slow.tokenizer					,							__SCREAMING_SNAKE_CASE			)
      self.assertIsInstance(processor_fast.tokenizer					,							__SCREAMING_SNAKE_CASE			)

      self.assertEqual(processor_slow.image_processor.to_json_string()					,							image_processor.to_json_string()			)
      self.assertEqual(processor_fast.image_processor.to_json_string()					,							image_processor.to_json_string()			)
      self.assertIsInstance(processor_slow.image_processor					,							__SCREAMING_SNAKE_CASE			)
      self.assertIsInstance(processor_fast.image_processor					,							__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       Optional[int]			)  -> Dict:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      CLIPProcessor(tokenizer=self.get_tokenizer()					,							image_processor=self.get_image_processor()			)
      processor.save_pretrained(self.tmpdirname			)

      __SCREAMING_SNAKE_CASE     =      self.get_tokenizer(bos_token="""(BOS)"""					,							eos_token="""(EOS)"""			)
      __SCREAMING_SNAKE_CASE     =      self.get_image_processor(do_normalize=__SCREAMING_SNAKE_CASE					,							padding_value=1.0			)

      __SCREAMING_SNAKE_CASE     =      CLIPProcessor.from_pretrained(
          self.tmpdirname					,							bos_token="""(BOS)"""					,							eos_token="""(EOS)"""					,							do_normalize=__SCREAMING_SNAKE_CASE					,							padding_value=1.0			)

      self.assertEqual(processor.tokenizer.get_vocab()					,							tokenizer_add_kwargs.get_vocab()			)
      self.assertIsInstance(processor.tokenizer					,							__SCREAMING_SNAKE_CASE			)

      self.assertEqual(processor.image_processor.to_json_string()					,							image_processor_add_kwargs.to_json_string()			)
      self.assertIsInstance(processor.image_processor					,							__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       List[str]			)  -> Dict:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.get_image_processor()
      __SCREAMING_SNAKE_CASE     =      self.get_tokenizer()

      __SCREAMING_SNAKE_CASE     =      CLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,							image_processor=__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      self.prepare_image_inputs()

      __SCREAMING_SNAKE_CASE     =      image_processor(__SCREAMING_SNAKE_CASE					,							return_tensors="""np"""			)
      __SCREAMING_SNAKE_CASE     =      processor(images=__SCREAMING_SNAKE_CASE					,							return_tensors="""np"""			)

      for key in input_image_proc.keys():
           self.assertAlmostEqual(input_image_proc[key].sum()					,							input_processor[key].sum()					,							delta=1E-2			)
 def       UpperCAmelCase__		(		self					:       List[Any]			)  -> int:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.get_image_processor()
      __SCREAMING_SNAKE_CASE     =      self.get_tokenizer()

      __SCREAMING_SNAKE_CASE     =      CLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,							image_processor=__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      """lower newer"""

      __SCREAMING_SNAKE_CASE     =      processor(text=__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      tokenizer(__SCREAMING_SNAKE_CASE			)

      for key in encoded_tok.keys():
           self.assertListEqual(encoded_tok[key]					,							encoded_processor[key]			)
 def       UpperCAmelCase__		(		self					:       Dict			)  -> Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.get_image_processor()
      __SCREAMING_SNAKE_CASE     =      self.get_tokenizer()

      __SCREAMING_SNAKE_CASE     =      CLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,							image_processor=__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      """lower newer"""
      __SCREAMING_SNAKE_CASE     =      self.prepare_image_inputs()

      __SCREAMING_SNAKE_CASE     =      processor(text=__SCREAMING_SNAKE_CASE					,							images=__SCREAMING_SNAKE_CASE			)

      self.assertListEqual(list(inputs.keys()			)					,							["""input_ids""", """attention_mask""", """pixel_values"""]			)

      # test if it raises when no input is passed
      with pytest.raises(__SCREAMING_SNAKE_CASE			):
           processor()
 def       UpperCAmelCase__		(		self					:       List[str]			)  -> Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.get_image_processor()
      __SCREAMING_SNAKE_CASE     =      self.get_tokenizer()

      __SCREAMING_SNAKE_CASE     =      CLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,							image_processor=__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

      __SCREAMING_SNAKE_CASE     =      processor.batch_decode(__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      tokenizer.batch_decode(__SCREAMING_SNAKE_CASE			)

      self.assertListEqual(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)


 def       UpperCAmelCase__		(		self					:       int			)  -> str:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.get_image_processor()
      __SCREAMING_SNAKE_CASE     =      self.get_tokenizer()

      __SCREAMING_SNAKE_CASE     =      CLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,							image_processor=__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      """lower newer"""
      __SCREAMING_SNAKE_CASE     =      self.prepare_image_inputs()

      __SCREAMING_SNAKE_CASE     =      processor(text=__SCREAMING_SNAKE_CASE					,							images=__SCREAMING_SNAKE_CASE			)

      self.assertListEqual(list(inputs.keys()			)					,							processor.model_input_names			)






