




"""simple docstring"""






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase__     						=						logging.get_logger(__name__)

lowercase__     						=						"""▁"""

lowercase__     						=						{"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

lowercase__     						=						{
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

lowercase__     						=						{
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

lowercase__     						=						{
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

lowercase__     						=						{
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}






class lowerCAmelCase__	(     lowercase						):






			'''simple docstring'''



			lowerCamelCase__      =			["input_ids"]

			lowerCamelCase__      =			VOCAB_FILES_NAMES
			lowerCamelCase__      =			PRETRAINED_INIT_CONFIGURATION
			lowerCamelCase__      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__      =			PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__      =			RESOURCE_FILES_NAMES
			def __init__(  self  ,      lowercase  ,      lowercase=None  ,      lowercase=False  ,      lowercase="utf8"  ,      lowercase="[UNK]"  ,      lowercase="[SEP]"  ,      lowercase="[PAD]"  ,      lowercase="[CLS]"  ,      lowercase="[MASK]"  ,      lowercase = None  ,      **lowercase  ,      ):
							# Mask token behave like a normal word, i.e. include the space before it and
							# is included in the raw text, there should be a match in a non-normalized sentence.

							_lowerCamelCase     :			Optional[int]    =    {} if sp_model_kwargs is None else sp_model_kwargs
							super().__init__(
							    do_lower_case=lowercase  ,      unk_token=lowercase  ,      sep_token=lowercase  ,      pad_token=lowercase  ,      cls_token=lowercase  ,      mask_token=lowercase  ,      vocab_file=lowercase  ,      encoding=lowercase  ,      sp_model_kwargs=self.sp_model_kwargs  ,      **lowercase  ,      )
							_lowerCamelCase     :			str    =    do_lower_case
							_lowerCamelCase     :			Optional[Any]    =    sentencepiece_model_ckpt
							_lowerCamelCase     :			int    =    spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.Load(lowercase  )

							# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
							if vocab_file is not None:
											_lowerCamelCase     :			Dict    =    self.load_vocab(filepath=lowercase  )
							else:
											_lowerCamelCase     :			Optional[int]    =    {self.sp_model.id_to_piece(lowercase  ): id for id in range(self.sp_model.get_piece_size()  )}
							_lowerCamelCase     :			int    =    {v: k for k, v in self.vocab.items()}
			def 					A_    (  self  ,      lowercase  ):
							if text is None:
											return None

							_lowerCamelCase     :			Tuple    =    self.tokenize(lowercase  )
							_lowerCamelCase, _lowerCamelCase     :			Any    =    '', []

							for i, ch in enumerate(lowercase  ):
											if ch in self.SP_CHAR_MAPPING:
															_lowerCamelCase     :			List[str]    =    self.SP_CHAR_MAPPING.get(lowercase  )
											else:
															_lowerCamelCase     :			Dict    =    unicodedata.normalize('NFKC'  ,      lowercase  )
											if self.is_whitespace(lowercase  ):
															continue
											normalized_text += ch
											char_mapping.extend([i] * len(lowercase  )  )

							_lowerCamelCase, _lowerCamelCase, _lowerCamelCase     :			Union[str, Any]    =    normalized_text, [], 0

							if self.do_lower_case:
											_lowerCamelCase     :			List[str]    =    text.lower()

							for token in split_tokens:
											if token[:1] == "▁":
															_lowerCamelCase     :			List[str]    =    token[1:]
											_lowerCamelCase     :			Union[str, Any]    =    text[offset:].index(lowercase  ) + offset
											_lowerCamelCase     :			Optional[int]    =    start + len(lowercase  )

											token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)  )
											_lowerCamelCase     :			Optional[Any]    =    end
							return token_mapping
			@property
			def 					A_    (  self  ):
							return len(self.vocab  )
			def 					A_    (  self  ):
							return dict(self.vocab  ,      **self.added_tokens_encoder  )
			def __getstate__(  self  ):
							_lowerCamelCase     :			str    =    self.__dict__.copy()
							_lowerCamelCase     :			Optional[int]    =    None
							return state
			def __setstate__(  self  ,      lowercase  ):
							_lowerCamelCase     :			Union[str, Any]    =    d

							# for backward compatibility
							if not hasattr(self  ,      'sp_model_kwargs'  ):
											_lowerCamelCase     :			List[str]    =    {}

							_lowerCamelCase     :			str    =    spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.Load(self.sentencepiece_model_ckpt  )
			def 					A_    (  self  ,      lowercase  ):
							return "".join((self.SP_CHAR_MAPPING.get(lowercase  ,      lowercase  ) for c in text)  )
			def 					A_    (  self  ,      lowercase  ,      lowercase=False  ,      lowercase=64  ,      lowercase=0.1  ):

							if self.sp_model_kwargs.get('enable_sampling'  ) is True:
											_lowerCamelCase     :			Optional[int]    =    True
							if self.sp_model_kwargs.get('alpha'  ) is not None:
											_lowerCamelCase     :			Tuple    =    self.sp_model_kwargs.get('alpha'  )
							if self.sp_model_kwargs.get('nbest_size'  ) is not None:
											_lowerCamelCase     :			Tuple    =    self.sp_model_kwargs.get('nbest_size'  )

							if not enable_sampling:
											_lowerCamelCase     :			Optional[Any]    =    self.sp_model.EncodeAsPieces(lowercase  )
							else:
											_lowerCamelCase     :			List[str]    =    self.sp_model.SampleEncodeAsPieces(lowercase  ,      lowercase  ,      lowercase  )
							_lowerCamelCase     :			Tuple    =    []
							for pi, piece in enumerate(lowercase  ):
											if piece == SPIECE_UNDERLINE:
															if not pieces[pi + 1].startswith(lowercase  ) and pi != 0:
																			new_pieces.append(lowercase  )
																			continue
															else:
																			continue
											_lowerCamelCase     :			Tuple    =    0
											for i, chunk in enumerate(lowercase  ):
															if chunk == SPIECE_UNDERLINE:
																			continue
															if self.is_ch_char(lowercase  ) or self.is_punct(lowercase  ):
																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																							new_pieces.append(piece[lst_i:i]  )
																			new_pieces.append(lowercase  )
																			_lowerCamelCase     :			int    =    i + 1
															elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																							new_pieces.append(piece[lst_i:i]  )
																			_lowerCamelCase     :			Any    =    i
															elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																							new_pieces.append(piece[lst_i:i]  )
																			_lowerCamelCase     :			Tuple    =    i
											if len(lowercase  ) > lst_i:
															new_pieces.append(piece[lst_i:]  )
							return new_pieces
			def 					A_    (  self  ,      lowercase  ):
							_lowerCamelCase     :			Optional[int]    =    ''.join(lowercase  ).replace(lowercase  ,      ' '  ).strip()
							return out_string
			def 					A_    (  self  ,      lowercase  ):
							_lowerCamelCase     :			Union[str, Any]    =    self.convert_ids_to_tokens(lowercase  )
							_lowerCamelCase     :			Any    =    ''.join(lowercase  ).replace(lowercase  ,      ' '  ).strip()
							return out_string
			def 					A_    (  self  ,      lowercase  ):
							return self.vocab.get(lowercase  ,      self.vocab.get(self.unk_token  )  )
			def 					A_    (  self  ,      lowercase  ):
							return self.reverse_vocab.get(lowercase  ,      self.unk_token  )
			def 					A_    (  self  ,      lowercase  ,      lowercase=None  ):
							if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							_lowerCamelCase     :			List[Any]    =    [self.cls_token_id]
							_lowerCamelCase     :			Optional[Any]    =    [self.sep_token_id]
							return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep
			def 					A_    (  self  ,      lowercase  ,      lowercase=None  ):
							if offset_mapping_a is None:
											return [(0, 0)] + offset_mapping_a + [(0, 0)]

							return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]
			def 					A_    (  self  ,      lowercase  ,      lowercase=None  ,      lowercase=False  ):

							if already_has_special_tokens:
											if token_ids_a is not None:
															raise ValueError(
															    'You should not supply a second sequence if the provided sequence of '
															    'ids is already formatted with special tokens for the model.'  )
											return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

							if token_ids_a is not None:
											return [1] + ([0] * len(lowercase  )) + [1, 1] + ([0] * len(lowercase  )) + [1]
							return [1] + ([0] * len(lowercase  )) + [1]
			def 					A_    (  self  ,      lowercase  ,      lowercase = None  ):
							# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
							if token_ids_a is None:
											# [CLS] X [SEP]
											return (len(lowercase  ) + 2) * [0]

							# [CLS] A [SEP] [SEP] B [SEP]
							return [0] * (len(lowercase  ) + 1) + [1] * (len(lowercase  ) + 3)
			def 					A_    (  self  ,      lowercase  ):
							if "\u4e00" <= char <= "\u9fff":
											return True
							return False
			def 					A_    (  self  ,      lowercase  ):
							if ("a" <= char <= "z") or ("A" <= char <= "Z"):
											return True
							return False
			def 					A_    (  self  ,      lowercase  ):
							if char in ",;:.?!~，；：。？！《》【】":
											return True
							return False
			def 					A_    (  self  ,      lowercase  ):
							if char == " " or char == "\t" or char == "\n" or char == "\r":
											return True
							if len(lowercase  ) == 1:
											_lowerCamelCase     :			Tuple    =    unicodedata.category(lowercase  )
											if cat == "Zs":
															return True
							return False
			def 					A_    (  self  ,      lowercase  ):
							_lowerCamelCase     :			Tuple    =    {}
							with io.open(lowercase  ,      'r'  ,      encoding='utf-8'  ) as f:
											for index, line in enumerate(lowercase  ):
															_lowerCamelCase     :			int    =    line.rstrip('\n'  )
															_lowerCamelCase     :			Optional[Any]    =    int(lowercase  )

							return token_to_idx






			def 					A_    (  self  ,      lowercase  ,      lowercase = None  ):
							_lowerCamelCase     :			Union[str, Any]    =    0
							if os.path.isdir(lowercase  ):
											_lowerCamelCase     :			List[Any]    =    os.path.join(
											    lowercase  ,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )
							else:
											_lowerCamelCase     :			Optional[int]    =    (filename_prefix + '-' if filename_prefix else '') + save_directory
							with open(lowercase  ,      'w'  ,      encoding='utf-8'  ) as writer:
											for token, token_index in sorted(self.vocab.items()  ,      key=lambda lowercase  : kv[1]  ):
															if index != token_index:
																			logger.warning(
																			    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																			    ' Please check that the vocabulary is not corrupted!'  )
																			_lowerCamelCase     :			Optional[Any]    =    token_index
															writer.write(token + '\n'  )
															index += 1

							_lowerCamelCase     :			List[str]    =    os.path.join(lowercase  ,      'sentencepiece.bpe.model'  )
							with open(lowercase  ,      'wb'  ) as fi:
											_lowerCamelCase     :			Dict    =    self.sp_model.serialized_model_proto()
											fi.write(lowercase  )

							return (vocab_file,)





"""simple docstring"""






from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
			from transformers import (
			    TFAutoModel,
			    TFAutoModelForCausalLM,
			    TFAutoModelForMaskedLM,
			    TFAutoModelForPreTraining,
			    TFAutoModelForQuestionAnswering,
			    TFAutoModelForSeqaSeqLM,
			    TFAutoModelForSequenceClassification,
			    TFAutoModelForTableQuestionAnswering,
			    TFAutoModelForTokenClassification,
			    TFAutoModelWithLMHead,
			    TFBertForMaskedLM,
			    TFBertForPreTraining,
			    TFBertForQuestionAnswering,
			    TFBertForSequenceClassification,
			    TFBertModel,
			    TFFunnelBaseModel,
			    TFFunnelModel,
			    TFGPTaLMHeadModel,
			    TFRobertaForMaskedLM,
			    TFTaForConditionalGeneration,
			    TFTapasForQuestionAnswering,
			)
			from transformers.models.auto.modeling_tf_auto import (
			    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
			    TF_MODEL_FOR_MASKED_LM_MAPPING,
			    TF_MODEL_FOR_PRETRAINING_MAPPING,
			    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
			    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
			    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
			    TF_MODEL_MAPPING,
			)
			from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST






class lowerCAmelCase__	(     lowercase						):






			'''simple docstring'''



			lowerCamelCase__      =			"""new-model"""


if is_tf_available():






			class lowerCAmelCase__	(     lowercase						):






						'''simple docstring'''



						lowerCamelCase__      =			NewModelConfig






@require_tf
class lowerCAmelCase__	(     unittest.TestCase						):






			'''simple docstring'''



			@slow
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[str]    =    'bert-base-cased'
							_lowerCamelCase     :			Union[str, Any]    =    AutoConfig.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							_lowerCamelCase     :			Union[str, Any]    =    TFAutoModel.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[str]    =    'bert-base-cased'
							_lowerCamelCase     :			int    =    AutoConfig.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							_lowerCamelCase     :			int    =    TFAutoModelForPreTraining.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			Dict    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			int    =    TFAutoModelForCausalLM.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			str    =    TFAutoModelForCausalLM.from_pretrained(lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			List[Any]    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			str    =    TFAutoModelWithLMHead.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			Tuple    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Optional[int]    =    TFAutoModelForMaskedLM.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			Tuple    =    TFAutoModelForMaskedLM.from_pretrained(lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			Optional[int]    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Tuple    =    TFAutoModelForSeqaSeqLM.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			Tuple    =    TFAutoModelForSeqaSeqLM.from_pretrained(lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							for model_name in ["bert-base-uncased"]:
											_lowerCamelCase     :			str    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Union[str, Any]    =    TFAutoModelForSequenceClassification.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							for model_name in ["bert-base-uncased"]:
											_lowerCamelCase     :			Optional[Any]    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			List[str]    =    TFAutoModelForQuestionAnswering.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			@require_tensorflow_probability
			def 					A_    (  self  ):
							for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
											_lowerCamelCase     :			Dict    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Dict    =    TFAutoModelForTableQuestionAnswering.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			List[Any]    =    TFAutoModelForTableQuestionAnswering.from_pretrained(
											    lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			def 					A_    (  self  ):
							_lowerCamelCase     :			int    =    TFAutoModelWithLMHead.from_pretrained(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
							self.assertEqual(model.num_parameters()  ,      14410  )
							self.assertEqual(model.num_parameters(only_trainable=lowercase  )  ,      14410  )
			def 					A_    (  self  ):
							_lowerCamelCase     :			Any    =    TFAutoModelWithLMHead.from_pretrained(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
							self.assertEqual(model.num_parameters()  ,      14410  )
							self.assertEqual(model.num_parameters(only_trainable=lowercase  )  ,      14410  )
			def 					A_    (  self  ):
							# For the auto model mapping, FunnelConfig has two models: FunnelModel and FunnelBaseModel
							_lowerCamelCase     :			List[str]    =    TFAutoModel.from_pretrained('sgugger/funnel-random-tiny'  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							_lowerCamelCase     :			Optional[int]    =    copy.deepcopy(model.config  )
							_lowerCamelCase     :			Dict    =    ['FunnelBaseModel']
							_lowerCamelCase     :			List[Any]    =    TFAutoModel.from_config(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							with tempfile.TemporaryDirectory() as tmp_dir:
											model.save_pretrained(lowercase  )
											_lowerCamelCase     :			Tuple    =    TFAutoModel.from_pretrained(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			def 					A_    (  self  ):
							try:
											AutoConfig.register('new-model'  ,      lowercase  )

											_lowerCamelCase     :			Tuple    =    [
											    TFAutoModel,
											    TFAutoModelForCausalLM,
											    TFAutoModelForMaskedLM,
											    TFAutoModelForPreTraining,
											    TFAutoModelForQuestionAnswering,
											    TFAutoModelForSequenceClassification,
											    TFAutoModelForTokenClassification,
											]

											for auto_class in auto_classes:
															with self.subTest(auto_class.__name__  ):
																			# Wrong config class will raise an error
																			with self.assertRaises(lowercase  ):
																							auto_class.register(lowercase  ,      lowercase  )
																			auto_class.register(lowercase  ,      lowercase  )
																			# Trying to register something existing in the Transformers library will raise an error
																			with self.assertRaises(lowercase  ):
																							auto_class.register(lowercase  ,      lowercase  )

																			# Now that the config is registered, it can be used as any other config with the auto-API
																			_lowerCamelCase     :			Optional[Any]    =    BertModelTester(self  ).get_config()
																			_lowerCamelCase     :			Dict    =    NewModelConfig(**tiny_config.to_dict()  )
																			_lowerCamelCase     :			int    =    auto_class.from_config(lowercase  )
																			self.assertIsInstance(lowercase  ,      lowercase  )

																			with tempfile.TemporaryDirectory() as tmp_dir:
																							model.save_pretrained(lowercase  )
																							_lowerCamelCase     :			List[Any]    =    auto_class.from_pretrained(lowercase  )
																							self.assertIsInstance(lowercase  ,      lowercase  )

							finally:
											if "new-model" in CONFIG_MAPPING._extra_content:
															del CONFIG_MAPPING._extra_content["new-model"]
											for mapping in (
											    TF_MODEL_MAPPING,
											    TF_MODEL_FOR_PRETRAINING_MAPPING,
											    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
											    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
											    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
											    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
											    TF_MODEL_FOR_MASKED_LM_MAPPING,
											):
															if NewModelConfig in mapping._extra_content:
																			del mapping._extra_content[NewModelConfig]
			def 					A_    (  self  ):
							with self.assertRaisesRegex(
							    lowercase  ,      'bert-base is not a local folder and is not a valid model identifier'  ):
											_lowerCamelCase     :			Union[str, Any]    =    TFAutoModel.from_pretrained('bert-base'  )
			def 					A_    (  self  ):
							with self.assertRaisesRegex(
							    lowercase  ,      r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'  ):
											_lowerCamelCase     :			str    =    TFAutoModel.from_pretrained(lowercase  ,      revision='aaaaaa'  )
			def 					A_    (  self  ):
							with self.assertRaisesRegex(
							    lowercase  ,      'hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'  ,      ):
											_lowerCamelCase     :			Tuple    =    TFAutoModel.from_pretrained('hf-internal-testing/config-no-model'  )
			def 					A_    (  self  ):
							with self.assertRaisesRegex(lowercase  ,      'Use `from_pt=True` to load this model'  ):
											_lowerCamelCase     :			Tuple    =    TFAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only'  )






			def 					A_    (  self  ):
							# Make sure we have cached the model.
							_lowerCamelCase     :			Optional[int]    =    TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'  )
							with RequestCounter() as counter:
											_lowerCamelCase     :			Optional[int]    =    TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'  )
											self.assertEqual(counter.get_request_count  ,      0  )
											self.assertEqual(counter.head_request_count  ,      1  )
											self.assertEqual(counter.other_request_count  ,      0  )

							# With a sharded checkpoint
							_lowerCamelCase     :			int    =    TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'  )
							with RequestCounter() as counter:
											_lowerCamelCase     :			List[Any]    =    TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'  )
											self.assertEqual(counter.get_request_count  ,      0  )
											self.assertEqual(counter.head_request_count  ,      1  )
											self.assertEqual(counter.other_request_count  ,      0  )