



import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


A__  :  int								=					get_tests_dir('''fixtures/test_sentencepiece.model''')

if is_sentencepiece_available():
						import sentencepiece as sp


A__  :  Dict								=					5
A__  :  str								=					10



@require_sentencepiece
@require_tokenizers
class 				__snake_case						(     UpperCamelCase_						,unittest.TestCase						):
							_a			      = SpeechaTextTokenizer
							_a			      = False
							_a			      = True




							def    UpperCAmelCase__			(       self     :       Dict):
									super().setUp()

									lowerCAmelCase_			:     Union[str, Any]									= sp.SentencePieceProcessor()
									spm_model.Load(A_)
									lowerCAmelCase_			:     Tuple									= ['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

									vocab += [spm_model.IdToPiece(id_) for id_ in range(len(A_))]
									lowerCAmelCase_			:     Tuple									= dict(zip(A_				,				range(len(A_))))

									lowerCAmelCase_			:     Union[str, Any]									= Path(self.tmpdirname)
									save_json(A_				,				save_dir / VOCAB_FILES_NAMES['''vocab_file'''])
									if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
											copyfile(A_				,				save_dir / VOCAB_FILES_NAMES['''spm_file'''])

									lowerCAmelCase_			:     List[Any]									= SpeechaTextTokenizer.from_pretrained(self.tmpdirname)
									tokenizer.save_pretrained(self.tmpdirname)




							def    UpperCAmelCase__			(       self     :       Dict):
									lowerCAmelCase_			:     Union[str, Any]									= '''<pad>'''
									lowerCAmelCase_			:     Union[str, Any]									= 1

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(A_)				,				A_)
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(A_)				,				A_)




							def    UpperCAmelCase__			(       self     :       List[str]):
									lowerCAmelCase_			:     Union[str, Any]									= list(self.get_tokenizer().get_vocab().keys())

									self.assertEqual(vocab_keys[0]				,				'''<s>''')
									self.assertEqual(vocab_keys[1]				,				'''<pad>''')
									self.assertEqual(vocab_keys[-1]				,				'''j''')
									self.assertEqual(len(A_)				,				1_0_0_1)




							def    UpperCAmelCase__			(       self     :       int):
									self.assertEqual(self.get_tokenizer().vocab_size				,				1_0_0_1)




							def    UpperCAmelCase__			(       self     :       Optional[Any]):
									lowerCAmelCase_			:     Union[str, Any]									= SpeechaTextTokenizer.from_pretrained(self.tmpdirname)

									lowerCAmelCase_			:     Optional[Any]									= tokenizer.tokenize('''This is a test''')
									self.assertListEqual(A_				,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(A_)				,				[2_8_9, 5_0, 1_4, 1_7_4, 3_8_6]				,				)

									lowerCAmelCase_			:     List[str]									= tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
									self.assertListEqual(
									    A_				,				[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.''']				,				)
									lowerCAmelCase_			:     Optional[Any]									= tokenizer.convert_tokens_to_ids(A_)
									self.assertListEqual(A_				,				[1_2, 2_5, 8_8, 5_9, 2_8, 2_3, 1_1, 4, 6_0_6, 3_5_1, 3_5_1, 3_5_1, 7, 1_6, 7_0, 5_0, 7_6, 8_4, 1_0, 4, 8])

									lowerCAmelCase_			:     Optional[Any]									= tokenizer.convert_ids_to_tokens(A_)
									self.assertListEqual(
									    A_				,				[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.''']				,				)







							@slow
							def    UpperCAmelCase__			(       self     :       int):
									# fmt: off
									lowerCAmelCase_			:     Union[str, Any]									= {'''input_ids''': [[3_7_9_1, 7_9_7, 3_1, 1_1, 6_4, 7_9_7, 3_1, 2_4_2_9, 4_3_3, 1_2, 1_1_7_6, 1_2, 2_0, 7_8_6, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 3_2_3_8, 7_9_7, 3_1, 1_1, 3_5, 9_3, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_7, 6_1_0, 4_0, 6_2, 4_5_5, 6_5_7, 1_0_4_2, 1_2_3, 7_8_0, 1_7_7, 3_7, 3_0_9, 2_4_1, 1_2_9_8, 5_1_4, 2_0, 2_9_2, 2_7_3_7, 1_1_4, 2_4_6_9, 2_4_1, 8_5, 6_4, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 4, 5_0_9, 4_0_6, 4_2_3, 3_7, 6_0_1, 4, 7_7_7, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 2_8_4, 4, 3_3_8_8, 5_1_1, 4_5_9, 4, 3_5_5_5, 4_0, 3_2_1, 3_0_2, 7_0_5, 4, 3_3_8_8, 5_1_1, 5_8_3, 3_2_6, 5, 5, 5, 6_2, 3_3_1_0, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 3_2, 3_1, 8_5_3, 4_1_8, 6_4, 5_8_3, 5_1_1, 1_6_0_5, 6_2, 3_5, 9_3, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 1_5_2_1, 6_4, 5_8_3, 5_1_1, 5_1_9, 6_2, 2_0, 1_5_1_5, 7_6_4, 2_0, 1_4_9, 2_6_1, 5_6_2_5, 7_9_7_2, 2_0, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_9_2_5, 1_6_7_5, 1_1, 1_5, 8_0_2, 7_9_7_2, 5_7_6, 2_1_7, 1_5_0_8, 1_1, 3_5, 9_3, 1_2_5_3, 2_4_4_1, 1_5, 2_8_9, 6_5_2, 3_1, 4_1_6, 3_2_1, 3_8_4_2, 1_1_5, 4_0, 9_1_1, 8, 4_7_6, 6_1_9, 4, 3_8_0, 1_4_2, 4_2_3, 3_3_5, 2_4_0, 3_5, 9_3, 2_6_4, 8, 1_1, 3_3_5, 5_6_9, 4_2_0, 1_6_3, 5, 2], [2_6_0, 5_4_8, 5_2_8, 4_2_3, 2_0, 4_5_1, 2_0, 2_6_8_1, 1_1_5_3, 3_4_3_4, 2_0, 5_5_4_0, 3_7, 5_6_7, 1_2_6, 1_2_5_3, 2_4_4_1, 3_3_7_6, 4_4_9, 2_1_0, 4_3_1, 1_5_6_3, 1_7_7, 7_6_7, 5_5_4_0, 1_1, 1_2_0_3, 4_7_2, 1_1, 2_9_5_3, 6_8_5, 2_8_5, 3_6_4, 7_0_6, 1_1_5_3, 2_0, 6_7_9_9, 2_0, 2_8_6_9, 2_0, 4_4_6_4, 1_2_6, 4_0, 2_4_2_9, 2_0, 1_0_4_0, 8_6_6, 2_6_6_4, 4_1_8, 2_0, 3_1_8, 2_0, 1_7_2_6, 1_8_6, 2_0, 2_6_5, 5_2_2, 3_5, 9_3, 2_1_9_1, 4_6_3_4, 2_0, 1_0_4_0, 1_2, 6_7_9_9, 1_5, 2_2_8, 2_3_5_6, 1_4_2, 3_1, 1_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_5_7_5, 2_6_6_6, 6_8_4, 1_5_8_2, 1_1_7_6, 1_2, 6_2_7, 1_4_9, 6_1_9, 2_0, 4_9_0_2, 5_6_3, 1_1, 2_0, 1_4_9, 2_6_1, 3_4_2_0, 2_3_5_6, 1_7_4, 1_4_2, 4_7_1_4, 1_3_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=A_				,				model_name='''facebook/s2t-small-mustc-en-de-st'''				,				revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad'''				,				)





@require_sentencepiece
class 				__snake_case						(     unittest.TestCase						):
							_a			      = '''valhalla/s2t_mustc_multilinguial_medium'''

							_a			      = '''C\'est trop cool'''
							_a			      = '''Esto es genial'''




							@classmethod
							def    UpperCAmelCase__			(       cls     :       Dict):
									lowerCAmelCase_			:     SpeechaTextTokenizer									= SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name)
									return cls




							def    UpperCAmelCase__			(       self     :       Optional[Any]):
									self.assertEqual(self.tokenizer.lang_code_to_id['''pt''']				,				4)
									self.assertEqual(self.tokenizer.lang_code_to_id['''ru''']				,				6)
									self.assertEqual(self.tokenizer.lang_code_to_id['''it''']				,				9)
									self.assertEqual(self.tokenizer.lang_code_to_id['''de''']				,				1_1)




							def    UpperCAmelCase__			(       self     :       Optional[Any]):
									self.assertEqual(self.tokenizer.vocab_size				,				1_0_0_0_0)




							def    UpperCAmelCase__			(       self     :       Dict):
									self.assertIn(A_				,				self.tokenizer.all_special_ids)
									lowerCAmelCase_			:     List[str]									= [ES_CODE, 4, 1_6_0_1, 4_7, 7_6_4_7, 2]
									lowerCAmelCase_			:     Optional[Any]									= self.tokenizer.decode(A_				,				skip_special_tokens=A_)
									lowerCAmelCase_			:     Union[str, Any]									= self.tokenizer.decode(generated_ids[1:]				,				skip_special_tokens=A_)
									self.assertEqual(A_				,				A_)
									self.assertNotIn(self.tokenizer.eos_token				,				A_)




							def    UpperCAmelCase__			(       self     :       Union[str, Any]):
									lowerCAmelCase_			:     Optional[int]									= '''fr'''
									lowerCAmelCase_			:     List[str]									= self.tokenizer(self.french_text).input_ids
									self.assertEqual(encoded[0]				,				A_)
									self.assertEqual(encoded[-1]				,				self.tokenizer.eos_token_id)







							def    UpperCAmelCase__			(       self     :       Any):
									lowerCAmelCase_			:     Any									= '''fr'''
									self.assertListEqual(self.tokenizer.prefix_tokens				,				[FR_CODE])

									lowerCAmelCase_			:     str									= '''es'''
									self.assertListEqual(self.tokenizer.prefix_tokens				,				[ES_CODE])










from ...configuration_utils import PretrainedConfig
from ...utils import logging


A__  :  List[str]								=					logging.get_logger(__name__)

A__  :  int								=					{
    '''weiweishi/roc-bert-base-zh''': '''https://huggingface.co/weiweishi/roc-bert-base-zh/resolve/main/config.json''',
}



class 				__snake_case						(     UpperCamelCase_						):
							_a			      = '''roc_bert'''




							def __init__(       self     :       List[Any]				,				A_     :       Optional[Any]=3_0_5_2_2				,				A_     :       List[str]=7_6_8				,				A_     :       Tuple=1_2				,				A_     :       List[str]=1_2				,				A_     :       List[str]=3_0_7_2				,				A_     :       Any="gelu"				,				A_     :       str=0.1				,				A_     :       int=0.1				,				A_     :       Optional[int]=5_1_2				,				A_     :       int=2				,				A_     :       List[str]=0.02				,				A_     :       Union[str, Any]=1e-12				,				A_     :       Union[str, Any]=True				,				A_     :       Tuple=0				,				A_     :       Union[str, Any]="absolute"				,				A_     :       Optional[Any]=None				,				A_     :       Any=True				,				A_     :       Optional[int]=True				,				A_     :       List[Any]=7_6_8				,				A_     :       str=9_1_0				,				A_     :       Dict=5_1_2				,				A_     :       Optional[int]=2_4_8_5_8				,				A_     :       Optional[Any]=True				,				**A_     :       Dict				,				):
									lowerCAmelCase_			:     List[str]									= vocab_size
									lowerCAmelCase_			:     Any									= max_position_embeddings
									lowerCAmelCase_			:     Dict									= hidden_size
									lowerCAmelCase_			:     Union[str, Any]									= num_hidden_layers
									lowerCAmelCase_			:     Tuple									= num_attention_heads
									lowerCAmelCase_			:     int									= intermediate_size
									lowerCAmelCase_			:     Tuple									= hidden_act
									lowerCAmelCase_			:     List[str]									= hidden_dropout_prob
									lowerCAmelCase_			:     Tuple									= attention_probs_dropout_prob
									lowerCAmelCase_			:     Any									= initializer_range
									lowerCAmelCase_			:     Union[str, Any]									= type_vocab_size
									lowerCAmelCase_			:     Union[str, Any]									= layer_norm_eps
									lowerCAmelCase_			:     Optional[int]									= use_cache
									lowerCAmelCase_			:     Tuple									= enable_pronunciation
									lowerCAmelCase_			:     Optional[Any]									= enable_shape
									lowerCAmelCase_			:     Union[str, Any]									= pronunciation_embed_dim
									lowerCAmelCase_			:     List[Any]									= pronunciation_vocab_size
									lowerCAmelCase_			:     Tuple									= shape_embed_dim
									lowerCAmelCase_			:     str									= shape_vocab_size
									lowerCAmelCase_			:     Optional[int]									= concat_input
									lowerCAmelCase_			:     Optional[Any]									= position_embedding_type
									lowerCAmelCase_			:     Optional[Any]									= classifier_dropout
									super().__init__(pad_token_id=A_				,				**A_)





