


'''simple docstring'''

import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast





class      snake_case    (    unittest.TestCase			):





  """simple docstring"""




  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  0






  @slow
  def   snake_case		(    self     ):







   """simple docstring"""





   for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
    lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A     )
    self.assertIsNotNone(__A     )
    self.assertIsInstance(__A			,  (BertTokenizer, BertTokenizerFast)     )
    self.assertGreater(len(__A     )			,  0     )

   for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
    lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A     )
    self.assertIsNotNone(__A     )
    self.assertIsInstance(__A			,  (GPTaTokenizer, GPTaTokenizerFast)     )
    self.assertGreater(len(__A     )			,  0     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A     )
   self.assertIsInstance(__A			,  (BertTokenizer, BertTokenizerFast)     )
   self.assertEqual(tokenizer.vocab_size			,  12     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A     )
   self.assertIsInstance(__A			,  (RobertaTokenizer, RobertaTokenizerFast)     )
   self.assertEqual(tokenizer.vocab_size			,  20     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  AutoConfig.from_pretrained(__A     )
   self.assertIsInstance(__A			,  __A     )
   # Check that tokenizer_type ≠ model_type
   lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A			,  config=__A     )
   self.assertIsInstance(__A			,  (BertTokenizer, BertTokenizerFast)     )
   self.assertEqual(tokenizer.vocab_size			,  12     )






  def   snake_case		(    self     ):







   """simple docstring"""





   with tempfile.TemporaryDirectory() as tmp_dir:
    shutil.copy("./tests/fixtures/vocab.txt"			,  os.path.join(__A			,  "vocab.txt"     )     )

    lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A			,  tokenizer_type="bert"			,  use_fast=__A     )
    self.assertIsInstance(__A			,  __A     )

   with tempfile.TemporaryDirectory() as tmp_dir:
    shutil.copy("./tests/fixtures/vocab.json"			,  os.path.join(__A			,  "vocab.json"     )     )
    shutil.copy("./tests/fixtures/merges.txt"			,  os.path.join(__A			,  "merges.txt"     )     )

    lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A			,  tokenizer_type="gpt2"			,  use_fast=__A     )
    self.assertIsInstance(__A			,  __A     )






  @require_tokenizers
  def   snake_case		(    self     ):







   """simple docstring"""





   with tempfile.TemporaryDirectory() as tmp_dir:
    shutil.copy("./tests/fixtures/vocab.txt"			,  os.path.join(__A			,  "vocab.txt"     )     )

    lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A			,  tokenizer_type="bert"     )
    self.assertIsInstance(__A			,  __A     )

   with tempfile.TemporaryDirectory() as tmp_dir:
    shutil.copy("./tests/fixtures/vocab.json"			,  os.path.join(__A			,  "vocab.json"     )     )
    shutil.copy("./tests/fixtures/merges.txt"			,  os.path.join(__A			,  "merges.txt"     )     )

    lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A			,  tokenizer_type="gpt2"     )
    self.assertIsInstance(__A			,  __A     )






  def   snake_case		(    self     ):







   """simple docstring"""





   with pytest.raises(__A     ):
    AutoTokenizer.from_pretrained("./"			,  tokenizer_type="xxx"     )






  @require_tokenizers
  def   snake_case		(    self     ):







   """simple docstring"""





   for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
    lowerCamelCase_      =  tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"     )
    self.assertIsInstance(__A			,  (BertTokenizer, BertTokenizerFast)     )

    if isinstance(__A			,  __A     ):
     self.assertEqual(tokenizer.basic_tokenizer.do_lower_case			,  __A     )
    else:
     self.assertEqual(tokenizer.do_lower_case			,  __A     )

    self.assertEqual(tokenizer.model_max_length			,  512     )






  @require_tokenizers
  def   snake_case		(    self     ):







   """simple docstring"""





   for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
    with self.assertRaisesRegex(
        __A			,  "julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"			,  ):
     lowerCamelCase_      =  tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  TOKENIZER_MAPPING.values()
   lowerCamelCase_      =  []

   for slow_tok, fast_tok in tokenizers:
    if slow_tok is not None:
     tokenizer_names.append(slow_tok.__name__     )

    if fast_tok is not None:
     tokenizer_names.append(fast_tok.__name__     )

   for tokenizer_name in tokenizer_names:
    # must find the right class
    tokenizer_class_from_name(__A     )






  @require_tokenizers
  def   snake_case		(    self     ):







   """simple docstring"""
   self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"			,  use_fast=__A     )			,  __A     )
   self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"     )			,  __A     )






  @require_tokenizers
  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  AutoTokenizer.from_pretrained("distilbert-base-uncased"			,  do_lower_case=__A     )
   lowerCamelCase_      =  "Hello, world. How are you?"
   lowerCamelCase_      =  tokenizer.tokenize(__A     )
   self.assertEqual("[UNK]"			,  tokens[0]     )

   lowerCamelCase_      =  AutoTokenizer.from_pretrained("microsoft/mpnet-base"			,  do_lower_case=__A     )
   lowerCamelCase_      =  tokenizer.tokenize(__A     )
   self.assertEqual("[UNK]"			,  tokens[0]     )






  @require_tokenizers
  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"     )
   self.assertEqual(type(__A     )			,  __A     )
   self.assertEqual(tokenizer.model_max_length			,  512     )
   self.assertEqual(tokenizer.vocab_size			,  3_0000     )
   self.assertEqual(tokenizer.unk_token			,  "[UNK]"     )
   self.assertEqual(tokenizer.padding_side			,  "right"     )
   self.assertEqual(tokenizer.truncation_side			,  "right"     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A     )
   self.assertIsInstance(__A			,  (BertTokenizer, BertTokenizerFast)     )
   with tempfile.TemporaryDirectory() as tmp_dir:
    tokenizer.save_pretrained(__A     )
    lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A     )

   self.assertIsInstance(__A			,  tokenizer.__class__     )
   self.assertEqual(tokenizera.vocab_size			,  12     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  AutoTokenizer.from_pretrained("ctrl"     )
   # There is no fast CTRL so this always gives us a slow tokenizer.
   self.assertIsInstance(__A			,  __A     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  get_tokenizer_config("bert-base-cased"     )
   lowerCamelCase_      =  config.pop("_commit_hash"			,  __A     )
   # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
   self.assertEqual(__A			,  {"do_lower_case": False}     )

   # This model does not have a tokenizer_config so we get back an empty dict.
   lowerCamelCase_      =  get_tokenizer_config(__A     )
   self.assertDictEqual(__A			,  {}     )

   # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
   lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A     )
   with tempfile.TemporaryDirectory() as tmp_dir:
    tokenizer.save_pretrained(__A     )
    lowerCamelCase_      =  get_tokenizer_config(__A     )

   # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
   self.assertEqual(config["tokenizer_class"]			,  "BertTokenizer"     )






  def   snake_case		(    self     ):







   """simple docstring"""





   try:
    AutoConfig.register("custom"			,  __A     )

    AutoTokenizer.register(__A			,  slow_tokenizer_class=__A     )
    # Trying to register something existing in the Transformers library will raise an error
    with self.assertRaises(__A     ):
     AutoTokenizer.register(__A			,  slow_tokenizer_class=__A     )

    lowerCamelCase_      =  CustomTokenizer.from_pretrained(__A     )
    with tempfile.TemporaryDirectory() as tmp_dir:
     tokenizer.save_pretrained(__A     )

     lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A     )
     self.assertIsInstance(__A			,  __A     )

   finally:
    if "custom" in CONFIG_MAPPING._extra_content:
     del CONFIG_MAPPING._extra_content["custom"]
    if CustomConfig in TOKENIZER_MAPPING._extra_content:
     del TOKENIZER_MAPPING._extra_content[CustomConfig]






  @require_tokenizers
  def   snake_case		(    self     ):







   """simple docstring"""





   try:
    AutoConfig.register("custom"			,  __A     )

    # Can register in two steps
    AutoTokenizer.register(__A			,  slow_tokenizer_class=__A     )
    self.assertEqual(TOKENIZER_MAPPING[CustomConfig]			,  (CustomTokenizer, None)     )
    AutoTokenizer.register(__A			,  fast_tokenizer_class=__A     )
    self.assertEqual(TOKENIZER_MAPPING[CustomConfig]			,  (CustomTokenizer, CustomTokenizerFast)     )

    del TOKENIZER_MAPPING._extra_content[CustomConfig]
    # Can register in one step
    AutoTokenizer.register(
        __A			,  slow_tokenizer_class=__A			,  fast_tokenizer_class=__A     )
    self.assertEqual(TOKENIZER_MAPPING[CustomConfig]			,  (CustomTokenizer, CustomTokenizerFast)     )

    # Trying to register something existing in the Transformers library will raise an error
    with self.assertRaises(__A     ):
     AutoTokenizer.register(__A			,  fast_tokenizer_class=__A     )

    # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
    # and that model does not have a tokenizer.json
    with tempfile.TemporaryDirectory() as tmp_dir:
     lowerCamelCase_      =  BertTokenizerFast.from_pretrained(__A     )
     bert_tokenizer.save_pretrained(__A     )
     lowerCamelCase_      =  CustomTokenizerFast.from_pretrained(__A     )

    with tempfile.TemporaryDirectory() as tmp_dir:
     tokenizer.save_pretrained(__A     )

     lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A     )
     self.assertIsInstance(__A			,  __A     )

     lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A			,  use_fast=__A     )
     self.assertIsInstance(__A			,  __A     )

   finally:
    if "custom" in CONFIG_MAPPING._extra_content:
     del CONFIG_MAPPING._extra_content["custom"]
    if CustomConfig in TOKENIZER_MAPPING._extra_content:
     del TOKENIZER_MAPPING._extra_content[CustomConfig]






  def   snake_case		(    self     ):







   """simple docstring"""





   with self.assertRaises(__A     ):
    lowerCamelCase_      =  AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"     )
   # If remote code is disabled, we can't load this config.
   with self.assertRaises(__A     ):
    lowerCamelCase_      =  AutoTokenizer.from_pretrained(
        "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=__A     )

   lowerCamelCase_      =  AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=__A     )
   self.assertTrue(tokenizer.special_attribute_present     )
   # Test tokenizer can be reloaded.
   with tempfile.TemporaryDirectory() as tmp_dir:
    tokenizer.save_pretrained(__A     )
    lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A			,  trust_remote_code=__A     )
   self.assertTrue(reloaded_tokenizer.special_attribute_present     )

   if is_tokenizers_available():
    self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"     )
    self.assertEqual(reloaded_tokenizer.__class__.__name__			,  "NewTokenizerFast"     )

    # Test we can also load the slow version
    lowerCamelCase_      =  AutoTokenizer.from_pretrained(
        "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=__A			,  use_fast=__A     )
    self.assertTrue(tokenizer.special_attribute_present     )
    self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"     )
    # Test tokenizer can be reloaded.
    with tempfile.TemporaryDirectory() as tmp_dir:
     tokenizer.save_pretrained(__A     )
     lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A			,  trust_remote_code=__A			,  use_fast=__A     )
    self.assertEqual(reloaded_tokenizer.__class__.__name__			,  "NewTokenizer"     )
    self.assertTrue(reloaded_tokenizer.special_attribute_present     )
   else:
    self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"     )
    self.assertEqual(reloaded_tokenizer.__class__.__name__			,  "NewTokenizer"     )






  @require_tokenizers
  def   snake_case		(    self     ):







   """simple docstring"""





   class      snake_case    (    lowercase			):





     """simple docstring"""




     _lowerCamelCase		    =  False





   class      snake_case    (    lowercase			):





     """simple docstring"""




     _lowerCamelCase		    =  NewTokenizer
     _lowerCamelCase		    =  False

   try:
    AutoConfig.register("custom"			,  __A     )
    AutoTokenizer.register(__A			,  slow_tokenizer_class=__A     )
    AutoTokenizer.register(__A			,  fast_tokenizer_class=__A     )
    # If remote code is not set, the default is to use local
    lowerCamelCase_      =  AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"     )
    self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"     )
    self.assertFalse(tokenizer.special_attribute_present     )
    lowerCamelCase_      =  AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"			,  use_fast=__A     )
    self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"     )
    self.assertFalse(tokenizer.special_attribute_present     )

    # If remote code is disabled, we load the local one.
    lowerCamelCase_      =  AutoTokenizer.from_pretrained(
        "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=__A     )
    self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"     )
    self.assertFalse(tokenizer.special_attribute_present     )
    lowerCamelCase_      =  AutoTokenizer.from_pretrained(
        "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=__A			,  use_fast=__A     )
    self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"     )
    self.assertFalse(tokenizer.special_attribute_present     )

    # If remote is enabled, we load from the Hub
    lowerCamelCase_      =  AutoTokenizer.from_pretrained(
        "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=__A     )
    self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"     )
    self.assertTrue(tokenizer.special_attribute_present     )
    lowerCamelCase_      =  AutoTokenizer.from_pretrained(
        "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=__A			,  use_fast=__A     )
    self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"     )
    self.assertTrue(tokenizer.special_attribute_present     )

   finally:
    if "custom" in CONFIG_MAPPING._extra_content:
     del CONFIG_MAPPING._extra_content["custom"]
    if CustomConfig in TOKENIZER_MAPPING._extra_content:
     del TOKENIZER_MAPPING._extra_content[CustomConfig]






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  AutoTokenizer.from_pretrained(
       "hf-internal-testing/test_dynamic_tokenizer_legacy"			,  trust_remote_code=__A     )
   self.assertTrue(tokenizer.special_attribute_present     )
   if is_tokenizers_available():
    self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"     )

    # Test we can also load the slow version
    lowerCamelCase_      =  AutoTokenizer.from_pretrained(
        "hf-internal-testing/test_dynamic_tokenizer_legacy"			,  trust_remote_code=__A			,  use_fast=__A     )
    self.assertTrue(tokenizer.special_attribute_present     )
    self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"     )
   else:
    self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"     )






  def   snake_case		(    self     ):







   """simple docstring"""





   with self.assertRaisesRegex(
       __A			,  "bert-base is not a local folder and is not a valid model identifier"     ):
    lowerCamelCase_      =  AutoTokenizer.from_pretrained("bert-base"     )






  def   snake_case		(    self     ):







   """simple docstring"""





   with self.assertRaisesRegex(
       __A			,  r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"     ):
    lowerCamelCase_      =  AutoTokenizer.from_pretrained(__A			,  revision="aaaaaa"     )







  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"     )
   with RequestCounter() as counter:
    lowerCamelCase_      =  AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"     )
    self.assertEqual(counter.get_request_count			,  0     )
    self.assertEqual(counter.head_request_count			,  1     )
    self.assertEqual(counter.other_request_count			,  0     )







def 		lowercase_(  SCREAMING_SNAKE_CASE_ = 4000000	):


  '''simple docstring'''

  lowerCamelCase :					Any    =				[0, 1]
  lowerCamelCase :					Union[str, Any]    =				0
  while fib[i] <= n:
    fib.append(fib[i] + fib[i + 1]	)
    if fib[i + 2] > n:
      break
    i += 1
  lowerCamelCase :					Union[str, Any]    =				0
  for j in range(len(SCREAMING_SNAKE_CASE_	) - 1	):
    if fib[j] % 2 == 0:
      total += fib[j]

  return total


if __name__ == "__main__":
   print(f'''{solution() = }''')



