


'''simple docstring'''







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A__:    List[str]					    =       logging.get_logger(__name__)

A__:    int					    =       {'''vocab_file''': '''spm_char.model'''}

A__:    Tuple					    =       {
    '''vocab_file''': {
        '''microsoft/speecht5_asr''': '''https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model''',
        '''microsoft/speecht5_tts''': '''https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model''',
        '''microsoft/speecht5_vc''': '''https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model''',
    }
}

A__:    Dict					    =       {
    '''microsoft/speecht5_asr''': 1024,
    '''microsoft/speecht5_tts''': 1024,
    '''microsoft/speecht5_vc''': 1024,
}







class A__       (						UpperCAmelCase__      ):
      __UpperCamelCase       :  Union[str, Any]	       =							VOCAB_FILES_NAMES
      __UpperCamelCase       :  Any	       =							PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase       :  Tuple	       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase       :  int	       =							["input_ids", "attention_mask"]






      def __init__( self			:Any     ,						SCREAMING_SNAKE_CASE			:Optional[Any]     ,						SCREAMING_SNAKE_CASE			:List[str]="<s>"     ,						SCREAMING_SNAKE_CASE			:Optional[int]="</s>"     ,						SCREAMING_SNAKE_CASE			:Tuple="<unk>"     ,						SCREAMING_SNAKE_CASE			:int="<pad>"     ,						SCREAMING_SNAKE_CASE			:Optional[Any] = None     ,						**SCREAMING_SNAKE_CASE			:int     ,						)       ->					Any:




             '''simple docstring'''
             _a    : List[str]     					={} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 bos_token=SCREAMING_SNAKE_CASE     ,						eos_token=SCREAMING_SNAKE_CASE     ,						unk_token=SCREAMING_SNAKE_CASE     ,						pad_token=SCREAMING_SNAKE_CASE     ,						sp_model_kwargs=self.sp_model_kwargs     ,						**SCREAMING_SNAKE_CASE     ,						)

             _a    : str     					=vocab_file

             _a    : Union[str, Any]     					=spm.SentencePieceProcessor(**self.sp_model_kwargs	)
             self.sp_model.Load(SCREAMING_SNAKE_CASE	)






      @property
      def __UpperCAmelCase       ( self			:Optional[Any]	)       ->					int:




             '''simple docstring'''
             return self.sp_model.get_piece_size()






      def __UpperCAmelCase       ( self			:List[Any]	)       ->					Any:




             '''simple docstring'''
             _a    : Optional[int]     					={self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE	): i for i in range(self.vocab_size	)}
             vocab.update(self.added_tokens_encoder	)
             return vocab






      def __getstate__( self			:int	)       ->					int:




             '''simple docstring'''
             _a    : Optional[Any]     					=self.__dict__.copy()
             _a    : Tuple     					=None
             return state






      def __setstate__( self			:Any     ,						SCREAMING_SNAKE_CASE			:Tuple	)       ->					Tuple:




             '''simple docstring'''
             _a    : int     					=d

             # for backward compatibility
             if not hasattr(self     ,						"""sp_model_kwargs"""	):
                    _a    : Tuple     					={}

             _a    : List[str]     					=spm.SentencePieceProcessor(**self.sp_model_kwargs	)
             self.sp_model.Load(self.vocab_file	)






      def __UpperCAmelCase       ( self			:Tuple     ,						SCREAMING_SNAKE_CASE			:Dict	)       ->					Union[str, Any]:




             '''simple docstring'''
             return self.sp_model.encode(SCREAMING_SNAKE_CASE     ,						out_type=SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:Any     ,						SCREAMING_SNAKE_CASE			:Dict	)       ->					List[str]:




             '''simple docstring'''
             return self.sp_model.piece_to_id(SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:Tuple     ,						SCREAMING_SNAKE_CASE			:Dict	)       ->					str:




             '''simple docstring'''
             _a    : Dict     					=self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE	)
             return token






      def __UpperCAmelCase       ( self			:int     ,						SCREAMING_SNAKE_CASE			:Any	)       ->					Union[str, Any]:




             '''simple docstring'''
             _a    : Dict     					=[]
             _a    : List[str]     					=''
             for token in tokens:
                    # make sure that special tokens are not decoded using sentencepiece model
                    if token in self.all_special_tokens:
                           out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE	) + token
                           _a    : List[Any]     					=[]
                    else:
                           current_sub_tokens.append(SCREAMING_SNAKE_CASE	)
             out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE	)
             return out_string.strip()






      def __UpperCAmelCase       ( self			:Optional[Any]     ,						SCREAMING_SNAKE_CASE			:Dict     ,						SCREAMING_SNAKE_CASE			:List[str]=None	)       ->					int:




             '''simple docstring'''
             if token_ids_a is None:
                    return token_ids_a + [self.eos_token_id]
             # We don't expect to process pairs, but leave the pair logic for API consistency
             return token_ids_a + token_ids_a + [self.eos_token_id]






      def __UpperCAmelCase       ( self			:List[str]     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]     ,						SCREAMING_SNAKE_CASE			:str = None     ,						SCREAMING_SNAKE_CASE			:Union[str, Any] = False	)       ->					Optional[Any]:




             '''simple docstring'''
             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=SCREAMING_SNAKE_CASE     ,						token_ids_a=SCREAMING_SNAKE_CASE     ,						already_has_special_tokens=SCREAMING_SNAKE_CASE	)

             _a    : Optional[int]     					=[1]
             if token_ids_a is None:
                    return ([0] * len(SCREAMING_SNAKE_CASE	)) + suffix_ones
             return ([0] * len(SCREAMING_SNAKE_CASE	)) + ([0] * len(SCREAMING_SNAKE_CASE	)) + suffix_ones






      def __UpperCAmelCase       ( self			:str     ,						SCREAMING_SNAKE_CASE			:List[Any]     ,						SCREAMING_SNAKE_CASE			:Union[str, Any] = None	)       ->					Dict:




             '''simple docstring'''
             if not os.path.isdir(SCREAMING_SNAKE_CASE	):
                    logger.error(f"Vocabulary path ({save_directory}) should be a directory"	)
                    return
             _a    : Tuple     					=os.path.join(
                 SCREAMING_SNAKE_CASE     ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)

             if os.path.abspath(self.vocab_file	) != os.path.abspath(SCREAMING_SNAKE_CASE	) and os.path.isfile(self.vocab_file	):
                    copyfile(self.vocab_file     ,						SCREAMING_SNAKE_CASE	)
             elif not os.path.isfile(self.vocab_file	):
                    with open(SCREAMING_SNAKE_CASE     ,						"""wb"""	) as fi:
                           _a    : str     					=self.sp_model.serialized_model_proto()
                           fi.write(SCREAMING_SNAKE_CASE	)

             return (out_vocab_file,)








"""simple docstring"""






from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
			from transformers import (
			    TFAutoModel,
			    TFAutoModelForCausalLM,
			    TFAutoModelForMaskedLM,
			    TFAutoModelForPreTraining,
			    TFAutoModelForQuestionAnswering,
			    TFAutoModelForSeqaSeqLM,
			    TFAutoModelForSequenceClassification,
			    TFAutoModelForTableQuestionAnswering,
			    TFAutoModelForTokenClassification,
			    TFAutoModelWithLMHead,
			    TFBertForMaskedLM,
			    TFBertForPreTraining,
			    TFBertForQuestionAnswering,
			    TFBertForSequenceClassification,
			    TFBertModel,
			    TFFunnelBaseModel,
			    TFFunnelModel,
			    TFGPTaLMHeadModel,
			    TFRobertaForMaskedLM,
			    TFTaForConditionalGeneration,
			    TFTapasForQuestionAnswering,
			)
			from transformers.models.auto.modeling_tf_auto import (
			    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
			    TF_MODEL_FOR_MASKED_LM_MAPPING,
			    TF_MODEL_FOR_PRETRAINING_MAPPING,
			    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
			    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
			    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
			    TF_MODEL_MAPPING,
			)
			from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST






class lowerCAmelCase__	(     lowercase						):






			'''simple docstring'''



			lowerCamelCase__      =			"""new-model"""


if is_tf_available():






			class lowerCAmelCase__	(     lowercase						):






						'''simple docstring'''



						lowerCamelCase__      =			NewModelConfig






@require_tf
class lowerCAmelCase__	(     unittest.TestCase						):






			'''simple docstring'''



			@slow
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[str]    =    'bert-base-cased'
							_lowerCamelCase     :			Union[str, Any]    =    AutoConfig.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							_lowerCamelCase     :			Union[str, Any]    =    TFAutoModel.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[str]    =    'bert-base-cased'
							_lowerCamelCase     :			int    =    AutoConfig.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							_lowerCamelCase     :			int    =    TFAutoModelForPreTraining.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			Dict    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			int    =    TFAutoModelForCausalLM.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			str    =    TFAutoModelForCausalLM.from_pretrained(lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			List[Any]    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			str    =    TFAutoModelWithLMHead.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			Tuple    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Optional[int]    =    TFAutoModelForMaskedLM.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			Tuple    =    TFAutoModelForMaskedLM.from_pretrained(lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			Optional[int]    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Tuple    =    TFAutoModelForSeqaSeqLM.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			Tuple    =    TFAutoModelForSeqaSeqLM.from_pretrained(lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							for model_name in ["bert-base-uncased"]:
											_lowerCamelCase     :			str    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Union[str, Any]    =    TFAutoModelForSequenceClassification.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							for model_name in ["bert-base-uncased"]:
											_lowerCamelCase     :			Optional[Any]    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			List[str]    =    TFAutoModelForQuestionAnswering.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			@require_tensorflow_probability
			def 					A_    (  self  ):
							for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
											_lowerCamelCase     :			Dict    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Dict    =    TFAutoModelForTableQuestionAnswering.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			List[Any]    =    TFAutoModelForTableQuestionAnswering.from_pretrained(
											    lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			def 					A_    (  self  ):
							_lowerCamelCase     :			int    =    TFAutoModelWithLMHead.from_pretrained(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
							self.assertEqual(model.num_parameters()  ,      14410  )
							self.assertEqual(model.num_parameters(only_trainable=lowercase  )  ,      14410  )
			def 					A_    (  self  ):
							_lowerCamelCase     :			Any    =    TFAutoModelWithLMHead.from_pretrained(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
							self.assertEqual(model.num_parameters()  ,      14410  )
							self.assertEqual(model.num_parameters(only_trainable=lowercase  )  ,      14410  )
			def 					A_    (  self  ):
							# For the auto model mapping, FunnelConfig has two models: FunnelModel and FunnelBaseModel
							_lowerCamelCase     :			List[str]    =    TFAutoModel.from_pretrained('sgugger/funnel-random-tiny'  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							_lowerCamelCase     :			Optional[int]    =    copy.deepcopy(model.config  )
							_lowerCamelCase     :			Dict    =    ['FunnelBaseModel']
							_lowerCamelCase     :			List[Any]    =    TFAutoModel.from_config(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							with tempfile.TemporaryDirectory() as tmp_dir:
											model.save_pretrained(lowercase  )
											_lowerCamelCase     :			Tuple    =    TFAutoModel.from_pretrained(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			def 					A_    (  self  ):
							try:
											AutoConfig.register('new-model'  ,      lowercase  )

											_lowerCamelCase     :			Tuple    =    [
											    TFAutoModel,
											    TFAutoModelForCausalLM,
											    TFAutoModelForMaskedLM,
											    TFAutoModelForPreTraining,
											    TFAutoModelForQuestionAnswering,
											    TFAutoModelForSequenceClassification,
											    TFAutoModelForTokenClassification,
											]

											for auto_class in auto_classes:
															with self.subTest(auto_class.__name__  ):
																			# Wrong config class will raise an error
																			with self.assertRaises(lowercase  ):
																							auto_class.register(lowercase  ,      lowercase  )
																			auto_class.register(lowercase  ,      lowercase  )
																			# Trying to register something existing in the Transformers library will raise an error
																			with self.assertRaises(lowercase  ):
																							auto_class.register(lowercase  ,      lowercase  )

																			# Now that the config is registered, it can be used as any other config with the auto-API
																			_lowerCamelCase     :			Optional[Any]    =    BertModelTester(self  ).get_config()
																			_lowerCamelCase     :			Dict    =    NewModelConfig(**tiny_config.to_dict()  )
																			_lowerCamelCase     :			int    =    auto_class.from_config(lowercase  )
																			self.assertIsInstance(lowercase  ,      lowercase  )

																			with tempfile.TemporaryDirectory() as tmp_dir:
																							model.save_pretrained(lowercase  )
																							_lowerCamelCase     :			List[Any]    =    auto_class.from_pretrained(lowercase  )
																							self.assertIsInstance(lowercase  ,      lowercase  )

							finally:
											if "new-model" in CONFIG_MAPPING._extra_content:
															del CONFIG_MAPPING._extra_content["new-model"]
											for mapping in (
											    TF_MODEL_MAPPING,
											    TF_MODEL_FOR_PRETRAINING_MAPPING,
											    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
											    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
											    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
											    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
											    TF_MODEL_FOR_MASKED_LM_MAPPING,
											):
															if NewModelConfig in mapping._extra_content:
																			del mapping._extra_content[NewModelConfig]
			def 					A_    (  self  ):
							with self.assertRaisesRegex(
							    lowercase  ,      'bert-base is not a local folder and is not a valid model identifier'  ):
											_lowerCamelCase     :			Union[str, Any]    =    TFAutoModel.from_pretrained('bert-base'  )
			def 					A_    (  self  ):
							with self.assertRaisesRegex(
							    lowercase  ,      r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'  ):
											_lowerCamelCase     :			str    =    TFAutoModel.from_pretrained(lowercase  ,      revision='aaaaaa'  )
			def 					A_    (  self  ):
							with self.assertRaisesRegex(
							    lowercase  ,      'hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'  ,      ):
											_lowerCamelCase     :			Tuple    =    TFAutoModel.from_pretrained('hf-internal-testing/config-no-model'  )
			def 					A_    (  self  ):
							with self.assertRaisesRegex(lowercase  ,      'Use `from_pt=True` to load this model'  ):
											_lowerCamelCase     :			Tuple    =    TFAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only'  )






			def 					A_    (  self  ):
							# Make sure we have cached the model.
							_lowerCamelCase     :			Optional[int]    =    TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'  )
							with RequestCounter() as counter:
											_lowerCamelCase     :			Optional[int]    =    TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'  )
											self.assertEqual(counter.get_request_count  ,      0  )
											self.assertEqual(counter.head_request_count  ,      1  )
											self.assertEqual(counter.other_request_count  ,      0  )

							# With a sharded checkpoint
							_lowerCamelCase     :			int    =    TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'  )
							with RequestCounter() as counter:
											_lowerCamelCase     :			List[Any]    =    TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'  )
											self.assertEqual(counter.get_request_count  ,      0  )
											self.assertEqual(counter.head_request_count  ,      1  )
											self.assertEqual(counter.other_request_count  ,      0  )