



'''simple docstring'''




import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
						from test_module.custom_tokenization_fast import CustomTokenizerFast







class 	_lowerCAmelCase							(			unittest.TestCase ):


	def     _a	(self							):
				A_	:						Tuple	=     0


	@slow
	def     _a	(self							):




				for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
							A_	:						List[Any]	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE							)
							self.assertIsNotNone(__SCREAMING_SNAKE_CASE							)
							self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      (BertTokenizer, BertTokenizerFast)							)
							self.assertGreater(len(__SCREAMING_SNAKE_CASE							)       ,      0							)

				for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
							A_	:						List[Any]	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE							)
							self.assertIsNotNone(__SCREAMING_SNAKE_CASE							)
							self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      (GPTaTokenizer, GPTaTokenizerFast)							)
							self.assertGreater(len(__SCREAMING_SNAKE_CASE							)       ,      0							)


	def     _a	(self							):
				A_	:						List[Any]	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE							)
				self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      (BertTokenizer, BertTokenizerFast)							)
				self.assertEqual(tokenizer.vocab_size       ,      12							)


	def     _a	(self							):
				A_	:						Tuple	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE							)
				self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      (RobertaTokenizer, RobertaTokenizerFast)							)
				self.assertEqual(tokenizer.vocab_size       ,      20							)


	def     _a	(self							):
				A_	:						int	=     AutoConfig.from_pretrained(__SCREAMING_SNAKE_CASE							)
				self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      __SCREAMING_SNAKE_CASE							)
				# Check that tokenizer_type ≠ model_type
				A_	:						int	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE       ,      config=__SCREAMING_SNAKE_CASE							)
				self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      (BertTokenizer, BertTokenizerFast)							)
				self.assertEqual(tokenizer.vocab_size       ,      12							)


	def     _a	(self							):




				with tempfile.TemporaryDirectory() as tmp_dir:
							shutil.copy("""./tests/fixtures/vocab.txt"""       ,      os.path.join(__SCREAMING_SNAKE_CASE       ,      """vocab.txt"""							)							)

							A_	:						Optional[Any]	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE       ,      tokenizer_type="""bert"""       ,      use_fast=__SCREAMING_SNAKE_CASE							)
							self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      __SCREAMING_SNAKE_CASE							)

				with tempfile.TemporaryDirectory() as tmp_dir:
							shutil.copy("""./tests/fixtures/vocab.json"""       ,      os.path.join(__SCREAMING_SNAKE_CASE       ,      """vocab.json"""							)							)
							shutil.copy("""./tests/fixtures/merges.txt"""       ,      os.path.join(__SCREAMING_SNAKE_CASE       ,      """merges.txt"""							)							)

							A_	:						int	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE       ,      tokenizer_type="""gpt2"""       ,      use_fast=__SCREAMING_SNAKE_CASE							)
							self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      __SCREAMING_SNAKE_CASE							)


	@require_tokenizers
	def     _a	(self							):




				with tempfile.TemporaryDirectory() as tmp_dir:
							shutil.copy("""./tests/fixtures/vocab.txt"""       ,      os.path.join(__SCREAMING_SNAKE_CASE       ,      """vocab.txt"""							)							)

							A_	:						Optional[int]	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE       ,      tokenizer_type="""bert"""							)
							self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      __SCREAMING_SNAKE_CASE							)

				with tempfile.TemporaryDirectory() as tmp_dir:
							shutil.copy("""./tests/fixtures/vocab.json"""       ,      os.path.join(__SCREAMING_SNAKE_CASE       ,      """vocab.json"""							)							)
							shutil.copy("""./tests/fixtures/merges.txt"""       ,      os.path.join(__SCREAMING_SNAKE_CASE       ,      """merges.txt"""							)							)

							A_	:						str	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE       ,      tokenizer_type="""gpt2"""							)
							self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      __SCREAMING_SNAKE_CASE							)


	def     _a	(self							):




				with pytest.raises(__SCREAMING_SNAKE_CASE							):
							AutoTokenizer.from_pretrained("""./"""       ,      tokenizer_type="""xxx"""							)


	@require_tokenizers
	def     _a	(self							):




				for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
							A_	:						int	=     tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""							)
							self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      (BertTokenizer, BertTokenizerFast)							)

							if isinstance(__SCREAMING_SNAKE_CASE       ,      __SCREAMING_SNAKE_CASE							):
										self.assertEqual(tokenizer.basic_tokenizer.do_lower_case       ,      __SCREAMING_SNAKE_CASE							)
							else:
										self.assertEqual(tokenizer.do_lower_case       ,      __SCREAMING_SNAKE_CASE							)

							self.assertEqual(tokenizer.model_max_length       ,      512							)


	@require_tokenizers
	def     _a	(self							):




				for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
							with self.assertRaisesRegex(
							    __SCREAMING_SNAKE_CASE       ,      """julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""       ,      ):
										A_	:						List[str]	=     tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""							)


	def     _a	(self							):
				A_	:						Dict	=     TOKENIZER_MAPPING.values()
				A_	:						Optional[int]	=     []

				for slow_tok, fast_tok in tokenizers:
							if slow_tok is not None:
										tokenizer_names.append(slow_tok.__name__							)

							if fast_tok is not None:
										tokenizer_names.append(fast_tok.__name__							)

				for tokenizer_name in tokenizer_names:
							# must find the right class
							tokenizer_class_from_name(__SCREAMING_SNAKE_CASE							)


	@require_tokenizers
	def     _a	(self							):
				self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""       ,      use_fast=__SCREAMING_SNAKE_CASE							)       ,      __SCREAMING_SNAKE_CASE							)
				self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""							)       ,      __SCREAMING_SNAKE_CASE							)


	@require_tokenizers
	def     _a	(self							):
				A_	:						List[Any]	=     AutoTokenizer.from_pretrained("""distilbert-base-uncased"""       ,      do_lower_case=__SCREAMING_SNAKE_CASE							)
				A_	:						Optional[int]	=     """Hello, world. How are you?"""
				A_	:						str	=     tokenizer.tokenize(__SCREAMING_SNAKE_CASE							)
				self.assertEqual("""[UNK]"""       ,      tokens[0]							)

				A_	:						Tuple	=     AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""       ,      do_lower_case=__SCREAMING_SNAKE_CASE							)
				A_	:						Optional[int]	=     tokenizer.tokenize(__SCREAMING_SNAKE_CASE							)
				self.assertEqual("""[UNK]"""       ,      tokens[0]							)


	@require_tokenizers
	def     _a	(self							):
				A_	:						str	=     AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""							)
				self.assertEqual(type(__SCREAMING_SNAKE_CASE							)       ,      __SCREAMING_SNAKE_CASE							)
				self.assertEqual(tokenizer.model_max_length       ,      512							)
				self.assertEqual(tokenizer.vocab_size       ,      30000							)
				self.assertEqual(tokenizer.unk_token       ,      """[UNK]"""							)
				self.assertEqual(tokenizer.padding_side       ,      """right"""							)
				self.assertEqual(tokenizer.truncation_side       ,      """right"""							)


	def     _a	(self							):
				A_	:						List[str]	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE							)
				self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      (BertTokenizer, BertTokenizerFast)							)
				with tempfile.TemporaryDirectory() as tmp_dir:
							tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE							)
							A_	:						List[str]	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE							)

				self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      tokenizer.__class__							)
				self.assertEqual(tokenizera.vocab_size       ,      12							)


	def     _a	(self							):
				A_	:						Any	=     AutoTokenizer.from_pretrained("""ctrl"""							)
				# There is no fast CTRL so this always gives us a slow tokenizer.
				self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      __SCREAMING_SNAKE_CASE							)


	def     _a	(self							):
				A_	:						Optional[Any]	=     get_tokenizer_config("""bert-base-cased"""							)
				A_	:						List[Any]	=     config.pop("""_commit_hash"""       ,      __SCREAMING_SNAKE_CASE							)
				# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
				self.assertEqual(__SCREAMING_SNAKE_CASE       ,      {"""do_lower_case""": False}							)

				# This model does not have a tokenizer_config so we get back an empty dict.
				A_	:						int	=     get_tokenizer_config(__SCREAMING_SNAKE_CASE							)
				self.assertDictEqual(__SCREAMING_SNAKE_CASE       ,      {}							)

				# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
				A_	:						Any	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE							)
				with tempfile.TemporaryDirectory() as tmp_dir:
							tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE							)
							A_	:						List[str]	=     get_tokenizer_config(__SCREAMING_SNAKE_CASE							)

				# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
				self.assertEqual(config["""tokenizer_class"""]       ,      """BertTokenizer"""							)


	def     _a	(self							):




				try:
							AutoConfig.register("""custom"""       ,      __SCREAMING_SNAKE_CASE							)

							AutoTokenizer.register(__SCREAMING_SNAKE_CASE       ,      slow_tokenizer_class=__SCREAMING_SNAKE_CASE							)
							# Trying to register something existing in the Transformers library will raise an error
							with self.assertRaises(__SCREAMING_SNAKE_CASE							):
										AutoTokenizer.register(__SCREAMING_SNAKE_CASE       ,      slow_tokenizer_class=__SCREAMING_SNAKE_CASE							)

							A_	:						str	=     CustomTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE							)
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE							)

										A_	:						Any	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE							)
										self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      __SCREAMING_SNAKE_CASE							)

				finally:
							if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
							if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]


	@require_tokenizers
	def     _a	(self							):




				try:
							AutoConfig.register("""custom"""       ,      __SCREAMING_SNAKE_CASE							)

							# Can register in two steps
							AutoTokenizer.register(__SCREAMING_SNAKE_CASE       ,      slow_tokenizer_class=__SCREAMING_SNAKE_CASE							)
							self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,      (CustomTokenizer, None)							)
							AutoTokenizer.register(__SCREAMING_SNAKE_CASE       ,      fast_tokenizer_class=__SCREAMING_SNAKE_CASE							)
							self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,      (CustomTokenizer, CustomTokenizerFast)							)

							del TOKENIZER_MAPPING._extra_content[CustomConfig]
							# Can register in one step
							AutoTokenizer.register(
							    __SCREAMING_SNAKE_CASE       ,      slow_tokenizer_class=__SCREAMING_SNAKE_CASE       ,      fast_tokenizer_class=__SCREAMING_SNAKE_CASE							)
							self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,      (CustomTokenizer, CustomTokenizerFast)							)

							# Trying to register something existing in the Transformers library will raise an error
							with self.assertRaises(__SCREAMING_SNAKE_CASE							):
										AutoTokenizer.register(__SCREAMING_SNAKE_CASE       ,      fast_tokenizer_class=__SCREAMING_SNAKE_CASE							)

							# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
							# and that model does not have a tokenizer.json
							with tempfile.TemporaryDirectory() as tmp_dir:
										A_	:						List[Any]	=     BertTokenizerFast.from_pretrained(__SCREAMING_SNAKE_CASE							)
										bert_tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE							)
										A_	:						List[Any]	=     CustomTokenizerFast.from_pretrained(__SCREAMING_SNAKE_CASE							)

							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE							)

										A_	:						Optional[Any]	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE							)
										self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      __SCREAMING_SNAKE_CASE							)

										A_	:						Optional[Any]	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE       ,      use_fast=__SCREAMING_SNAKE_CASE							)
										self.assertIsInstance(__SCREAMING_SNAKE_CASE       ,      __SCREAMING_SNAKE_CASE							)

				finally:
							if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
							if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]


	def     _a	(self							):




				with self.assertRaises(__SCREAMING_SNAKE_CASE							):
							A_	:						Any	=     AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
				# If remote code is disabled, we can't load this config.
				with self.assertRaises(__SCREAMING_SNAKE_CASE							):
							A_	:						Optional[int]	=     AutoTokenizer.from_pretrained(
							    """hf-internal-testing/test_dynamic_tokenizer"""       ,      trust_remote_code=__SCREAMING_SNAKE_CASE							)

				A_	:						Any	=     AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""       ,      trust_remote_code=__SCREAMING_SNAKE_CASE							)
				self.assertTrue(tokenizer.special_attribute_present							)
				# Test tokenizer can be reloaded.
				with tempfile.TemporaryDirectory() as tmp_dir:
							tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE							)
							A_	:						Optional[Any]	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE       ,      trust_remote_code=__SCREAMING_SNAKE_CASE							)
				self.assertTrue(reloaded_tokenizer.special_attribute_present							)

				if is_tokenizers_available():
							self.assertEqual(tokenizer.__class__.__name__       ,      """NewTokenizerFast"""							)
							self.assertEqual(reloaded_tokenizer.__class__.__name__       ,      """NewTokenizerFast"""							)

							# Test we can also load the slow version
							A_	:						Union[str, Any]	=     AutoTokenizer.from_pretrained(
							    """hf-internal-testing/test_dynamic_tokenizer"""       ,      trust_remote_code=__SCREAMING_SNAKE_CASE       ,      use_fast=__SCREAMING_SNAKE_CASE							)
							self.assertTrue(tokenizer.special_attribute_present							)
							self.assertEqual(tokenizer.__class__.__name__       ,      """NewTokenizer"""							)
							# Test tokenizer can be reloaded.
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE							)
										A_	:						Optional[int]	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE       ,      trust_remote_code=__SCREAMING_SNAKE_CASE       ,      use_fast=__SCREAMING_SNAKE_CASE							)
							self.assertEqual(reloaded_tokenizer.__class__.__name__       ,      """NewTokenizer"""							)
							self.assertTrue(reloaded_tokenizer.special_attribute_present							)
				else:
							self.assertEqual(tokenizer.__class__.__name__       ,      """NewTokenizer"""							)
							self.assertEqual(reloaded_tokenizer.__class__.__name__       ,      """NewTokenizer"""							)


	@require_tokenizers
	def     _a	(self							):







				class 	_lowerCAmelCase							(			__UpperCAmelCase ):
					__SCREAMING_SNAKE_CASE   :		Optional[Any]					=					False







				class 	_lowerCAmelCase							(			__UpperCAmelCase ):
					__SCREAMING_SNAKE_CASE   :		Any					=					NewTokenizer
					__SCREAMING_SNAKE_CASE   :		Optional[int]					=					False

				try:
							AutoConfig.register("""custom"""       ,      __SCREAMING_SNAKE_CASE							)
							AutoTokenizer.register(__SCREAMING_SNAKE_CASE       ,      slow_tokenizer_class=__SCREAMING_SNAKE_CASE							)
							AutoTokenizer.register(__SCREAMING_SNAKE_CASE       ,      fast_tokenizer_class=__SCREAMING_SNAKE_CASE							)
							# If remote code is not set, the default is to use local
							A_	:						List[Any]	=     AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
							self.assertEqual(tokenizer.__class__.__name__       ,      """NewTokenizerFast"""							)
							self.assertFalse(tokenizer.special_attribute_present							)
							A_	:						Optional[int]	=     AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""       ,      use_fast=__SCREAMING_SNAKE_CASE							)
							self.assertEqual(tokenizer.__class__.__name__       ,      """NewTokenizer"""							)
							self.assertFalse(tokenizer.special_attribute_present							)

							# If remote code is disabled, we load the local one.
							A_	:						Union[str, Any]	=     AutoTokenizer.from_pretrained(
							    """hf-internal-testing/test_dynamic_tokenizer"""       ,      trust_remote_code=__SCREAMING_SNAKE_CASE							)
							self.assertEqual(tokenizer.__class__.__name__       ,      """NewTokenizerFast"""							)
							self.assertFalse(tokenizer.special_attribute_present							)
							A_	:						Optional[Any]	=     AutoTokenizer.from_pretrained(
							    """hf-internal-testing/test_dynamic_tokenizer"""       ,      trust_remote_code=__SCREAMING_SNAKE_CASE       ,      use_fast=__SCREAMING_SNAKE_CASE							)
							self.assertEqual(tokenizer.__class__.__name__       ,      """NewTokenizer"""							)
							self.assertFalse(tokenizer.special_attribute_present							)

							# If remote is enabled, we load from the Hub
							A_	:						List[Any]	=     AutoTokenizer.from_pretrained(
							    """hf-internal-testing/test_dynamic_tokenizer"""       ,      trust_remote_code=__SCREAMING_SNAKE_CASE							)
							self.assertEqual(tokenizer.__class__.__name__       ,      """NewTokenizerFast"""							)
							self.assertTrue(tokenizer.special_attribute_present							)
							A_	:						Optional[Any]	=     AutoTokenizer.from_pretrained(
							    """hf-internal-testing/test_dynamic_tokenizer"""       ,      trust_remote_code=__SCREAMING_SNAKE_CASE       ,      use_fast=__SCREAMING_SNAKE_CASE							)
							self.assertEqual(tokenizer.__class__.__name__       ,      """NewTokenizer"""							)
							self.assertTrue(tokenizer.special_attribute_present							)

				finally:
							if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
							if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]


	def     _a	(self							):
				A_	:						List[Any]	=     AutoTokenizer.from_pretrained(
				    """hf-internal-testing/test_dynamic_tokenizer_legacy"""       ,      trust_remote_code=__SCREAMING_SNAKE_CASE							)
				self.assertTrue(tokenizer.special_attribute_present							)
				if is_tokenizers_available():
							self.assertEqual(tokenizer.__class__.__name__       ,      """NewTokenizerFast"""							)

							# Test we can also load the slow version
							A_	:						Any	=     AutoTokenizer.from_pretrained(
							    """hf-internal-testing/test_dynamic_tokenizer_legacy"""       ,      trust_remote_code=__SCREAMING_SNAKE_CASE       ,      use_fast=__SCREAMING_SNAKE_CASE							)
							self.assertTrue(tokenizer.special_attribute_present							)
							self.assertEqual(tokenizer.__class__.__name__       ,      """NewTokenizer"""							)
				else:
							self.assertEqual(tokenizer.__class__.__name__       ,      """NewTokenizer"""							)


	def     _a	(self							):




				with self.assertRaisesRegex(
				    __SCREAMING_SNAKE_CASE       ,      """bert-base is not a local folder and is not a valid model identifier"""							):
							A_	:						Tuple	=     AutoTokenizer.from_pretrained("""bert-base"""							)


	def     _a	(self							):




				with self.assertRaisesRegex(
				    __SCREAMING_SNAKE_CASE       ,      R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""							):
							A_	:						str	=     AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE       ,      revision="""aaaaaa"""							)


	def     _a	(self							):
				A_	:						List[Any]	=     AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
				with RequestCounter() as counter:
							A_	:						str	=     AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
							self.assertEqual(counter.get_request_count       ,      0							)
							self.assertEqual(counter.head_request_count       ,      1							)
							self.assertEqual(counter.other_request_count       ,      0							)


'''simple docstring'''



import math
from enum import Enum
from typing import Optional, Union

from torch.optim import Optimizer
from torch.optim.lr_scheduler import LambdaLR

from .utils import logging


UpperCAmelCase :       str			   =     logging.get_logger(__name__)







class 						lowerCAmelCase__   (      a					):



 """simple docstring"""



 lowerCAmelCase__  					=      "linear"
 lowerCAmelCase__  					=      "cosine"
 lowerCAmelCase__  					=      "cosine_with_restarts"
 lowerCAmelCase__  					=      "polynomial"
 lowerCAmelCase__  					=      "constant"
 lowerCAmelCase__  					=      "constant_with_warmup"
 lowerCAmelCase__  					=      "piecewise_constant"







def    a__       (  a__      ,    a__ = -1						):




     """simple docstring"""


     return LambdaLR(a__      ,    lambda a__						: 1      ,    last_epoch=a__						)







def    a__       (  a__      ,    a__      ,    a__ = -1						):




     """simple docstring"""


     def lr_lambda(a__						):
          if current_step < num_warmup_steps:
               return float(a__						) / float(max(1.0      ,    a__						)						)
          return 1.0

     return LambdaLR(a__      ,    a__      ,    last_epoch=a__						)







def    a__       (  a__      ,    a__      ,    a__ = -1						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      {}
     __SCREAMING_SNAKE_CASE     =      step_rules.split(""","""						)
     for rule_str in rule_list[:-1]:
          __SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE     =      rule_str.split(""":"""						)
          __SCREAMING_SNAKE_CASE     =      int(a__						)
          __SCREAMING_SNAKE_CASE     =      float(a__						)
          __SCREAMING_SNAKE_CASE     =      value
     __SCREAMING_SNAKE_CASE     =      float(rule_list[-1]						)

     def create_rules_function(a__      ,    a__						):
          def rule_func(a__						) -> float:
               __SCREAMING_SNAKE_CASE     =      sorted(rules_dict.keys()						)
               for i, sorted_step in enumerate(a__						):
                    if steps < sorted_step:
                         return rules_dict[sorted_steps[i]]
               return last_lr_multiple

          return rule_func

     __SCREAMING_SNAKE_CASE     =      create_rules_function(a__      ,    a__						)

     return LambdaLR(a__      ,    a__      ,    last_epoch=a__						)







def    a__       (  a__      ,    a__      ,    a__      ,    a__=-1						):




     """simple docstring"""


     def lr_lambda(a__						):
          if current_step < num_warmup_steps:
               return float(a__						) / float(max(1      ,    a__						)						)
          return max(
              0.0      ,    float(num_training_steps - current_step						) / float(max(1      ,    num_training_steps - num_warmup_steps						)						)						)

     return LambdaLR(a__      ,    a__      ,    a__						)







def    a__       (  a__      ,    a__      ,    a__      ,    a__ = 0.5      ,    a__ = -1						):




     """simple docstring"""


     def lr_lambda(a__						):
          if current_step < num_warmup_steps:
               return float(a__						) / float(max(1      ,    a__						)						)
          __SCREAMING_SNAKE_CASE     =      float(current_step - num_warmup_steps						) / float(max(1      ,    num_training_steps - num_warmup_steps						)						)
          return max(0.0      ,    0.5 * (1.0 + math.cos(math.pi * float(a__						) * 2.0 * progress						))						)

     return LambdaLR(a__      ,    a__      ,    a__						)







def    a__       (  a__      ,    a__      ,    a__      ,    a__ = 1      ,    a__ = -1						):




     """simple docstring"""


     def lr_lambda(a__						):
          if current_step < num_warmup_steps:
               return float(a__						) / float(max(1      ,    a__						)						)
          __SCREAMING_SNAKE_CASE     =      float(current_step - num_warmup_steps						) / float(max(1      ,    num_training_steps - num_warmup_steps						)						)
          if progress >= 1.0:
               return 0.0
          return max(0.0      ,    0.5 * (1.0 + math.cos(math.pi * ((float(a__						) * progress) % 1.0)						))						)

     return LambdaLR(a__      ,    a__      ,    a__						)







def    a__       (  a__      ,    a__      ,    a__      ,    a__=1E-7      ,    a__=1.0      ,    a__=-1						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      optimizer.defaults["""lr"""]
     if not (lr_init > lr_end):
          raise ValueError(F'lr_end ({lr_end}) must be be smaller than initial lr ({lr_init})'						)

     def lr_lambda(a__						):
          if current_step < num_warmup_steps:
               return float(a__						) / float(max(1      ,    a__						)						)
          elif current_step > num_training_steps:
               return lr_end / lr_init  # as LambdaLR multiplies by lr_init
          else:
               __SCREAMING_SNAKE_CASE     =      lr_init - lr_end
               __SCREAMING_SNAKE_CASE     =      num_training_steps - num_warmup_steps
               __SCREAMING_SNAKE_CASE     =      1 - (current_step - num_warmup_steps) / decay_steps
               __SCREAMING_SNAKE_CASE     =      lr_range * pct_remaining**power + lr_end
               return decay / lr_init  # as LambdaLR multiplies by lr_init

     return LambdaLR(a__      ,    a__      ,    a__						)


UpperCAmelCase :       Optional[Any]			   =     {
    SchedulerType.LINEAR: get_linear_schedule_with_warmup,
    SchedulerType.COSINE: get_cosine_schedule_with_warmup,
    SchedulerType.COSINE_WITH_RESTARTS: get_cosine_with_hard_restarts_schedule_with_warmup,
    SchedulerType.POLYNOMIAL: get_polynomial_decay_schedule_with_warmup,
    SchedulerType.CONSTANT: get_constant_schedule,
    SchedulerType.CONSTANT_WITH_WARMUP: get_constant_schedule_with_warmup,
    SchedulerType.PIECEWISE_CONSTANT: get_piecewise_constant_schedule,
}







def    a__       (  a__      ,    a__      ,    a__ = None      ,    a__ = None      ,    a__ = None      ,    a__ = 1      ,    a__ = 1.0      ,    a__ = -1      ,    ):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      SchedulerType(a__						)
     __SCREAMING_SNAKE_CASE     =      TYPE_TO_SCHEDULER_FUNCTION[name]
     if name == SchedulerType.CONSTANT:
          return schedule_func(a__      ,    last_epoch=a__						)

     if name == SchedulerType.PIECEWISE_CONSTANT:
          return schedule_func(a__      ,    step_rules=a__      ,    last_epoch=a__						)

     # All other schedulers require `num_warmup_steps`
     if num_warmup_steps is None:
          raise ValueError(F'{name} requires `num_warmup_steps`, please provide that argument.'						)

     if name == SchedulerType.CONSTANT_WITH_WARMUP:
          return schedule_func(a__      ,    num_warmup_steps=a__      ,    last_epoch=a__						)

     # All other schedulers require `num_training_steps`
     if num_training_steps is None:
          raise ValueError(F'{name} requires `num_training_steps`, please provide that argument.'						)

     if name == SchedulerType.COSINE_WITH_RESTARTS:
          return schedule_func(
              a__      ,    num_warmup_steps=a__      ,    num_training_steps=a__      ,    num_cycles=a__      ,    last_epoch=a__      ,    )

     if name == SchedulerType.POLYNOMIAL:
          return schedule_func(
              a__      ,    num_warmup_steps=a__      ,    num_training_steps=a__      ,    power=a__      ,    last_epoch=a__      ,    )

     return schedule_func(
         a__      ,    num_warmup_steps=a__      ,    num_training_steps=a__      ,    last_epoch=a__						)






