



"""simple docstring"""






import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case							:   Any      =    get_tests_dir('fixtures/test_sentencepiece.model')

if is_sentencepiece_available():
	import sentencepiece as sp


__snake_case							:   str      =    5
__snake_case							:   Optional[int]      =    10



@require_sentencepiece
@require_tokenizers
class 			A__   (  __SCREAMING_SNAKE_CASE  ,		unittest.TestCase							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE          =	SpeechaTextTokenizer
	SCREAMING_SNAKE_CASE          =	False
	SCREAMING_SNAKE_CASE          =	True

	def 							_SCREAMING_SNAKE_CASE   (     self:		Any)				->       str:

				"""simple docstring"""

				super().setUp()

				__lowerCAmelCase      :  Any												=       sp.SentencePieceProcessor()
				spm_model.Load(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase      :  Tuple												=       ["<s>", "<pad>", "</s>", "<unk>"]

				vocab += [spm_model.IdToPiece(id_) for id_ in range(len(_SCREAMING_SNAKE_CASE))]
				__lowerCAmelCase      :  Dict												=       dict(zip(_SCREAMING_SNAKE_CASE       ,			range(len(_SCREAMING_SNAKE_CASE))))

				__lowerCAmelCase      :  Any												=       Path(self.tmpdirname)
				save_json(_SCREAMING_SNAKE_CASE       ,			save_dir / VOCAB_FILES_NAMES["vocab_file"])
				if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
							copyfile(_SCREAMING_SNAKE_CASE       ,			save_dir / VOCAB_FILES_NAMES["spm_file"])

				__lowerCAmelCase      :  Optional[int]												=       SpeechaTextTokenizer.from_pretrained(self.tmpdirname)
				tokenizer.save_pretrained(self.tmpdirname)

	def 							_SCREAMING_SNAKE_CASE   (     self:		str)				->       List[Any]:

				"""simple docstring"""

				__lowerCAmelCase      :  List[Any]												=       "<pad>"
				__lowerCAmelCase      :  str												=       1

				self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE)       ,			_SCREAMING_SNAKE_CASE)
				self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE)       ,			_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE   (     self:		Union[str, Any])				->       Any:

				"""simple docstring"""

				__lowerCAmelCase      :  Union[str, Any]												=       list(self.get_tokenizer().get_vocab().keys())

				self.assertEqual(vocab_keys[0]       ,			"<s>")
				self.assertEqual(vocab_keys[1]       ,			"<pad>")
				self.assertEqual(vocab_keys[-1]       ,			"j")
				self.assertEqual(len(_SCREAMING_SNAKE_CASE)       ,			1001)

	def 							_SCREAMING_SNAKE_CASE   (     self:		int)				->       str:

				"""simple docstring"""

				self.assertEqual(self.get_tokenizer().vocab_size       ,			1001)

	def 							_SCREAMING_SNAKE_CASE   (     self:		Union[str, Any])				->       Any:

				"""simple docstring"""

				__lowerCAmelCase      :  Dict												=       SpeechaTextTokenizer.from_pretrained(self.tmpdirname)

				__lowerCAmelCase      :  List[str]												=       tokenizer.tokenize("This is a test")
				self.assertListEqual(_SCREAMING_SNAKE_CASE       ,			["▁This", "▁is", "▁a", "▁t", "est"])

				self.assertListEqual(
				    tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE)       ,			[289, 50, 14, 174, 386]       ,			)

				__lowerCAmelCase      :  Any												=       tokenizer.tokenize("I was born in 92000, and this is falsé.")
				self.assertListEqual(
				    _SCREAMING_SNAKE_CASE       ,			[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "9", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "é", "."]       ,			)
				__lowerCAmelCase      :  Tuple												=       tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE)
				self.assertListEqual(_SCREAMING_SNAKE_CASE       ,			[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8])

				__lowerCAmelCase      :  List[str]												=       tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE)
				self.assertListEqual(
				    _SCREAMING_SNAKE_CASE       ,			[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "<unk>", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "<unk>", "."]       ,			)







	@slow
	def 							_SCREAMING_SNAKE_CASE   (     self:		Union[str, Any])				->       List[Any]:

				"""simple docstring"""

				__lowerCAmelCase      :  Tuple												=       {"input_ids": [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
				# fmt: on

				self.tokenizer_integration_test_util(
				    expected_encoding=_SCREAMING_SNAKE_CASE       ,			model_name="facebook/s2t-small-mustc-en-de-st"       ,			revision="a14f04cf0776c02f62a8cb800cf7909e15ea23ad"       ,			)






@require_sentencepiece
class 			A__   (  unittest.TestCase							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE          =	'valhalla/s2t_mustc_multilinguial_medium'

	SCREAMING_SNAKE_CASE          =	'C\'est trop cool'
	SCREAMING_SNAKE_CASE          =	'Esto es genial'

	@classmethod
	def 							_SCREAMING_SNAKE_CASE   (     cls:		str)				->       Any:

				"""simple docstring"""

				__lowerCAmelCase      :  SpeechaTextTokenizer												=       SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name)
				return cls

	def 							_SCREAMING_SNAKE_CASE   (     self:		Dict)				->       str:

				"""simple docstring"""

				self.assertEqual(self.tokenizer.lang_code_to_id["pt"]       ,			4)
				self.assertEqual(self.tokenizer.lang_code_to_id["ru"]       ,			6)
				self.assertEqual(self.tokenizer.lang_code_to_id["it"]       ,			9)
				self.assertEqual(self.tokenizer.lang_code_to_id["de"]       ,			11)

	def 							_SCREAMING_SNAKE_CASE   (     self:		Union[str, Any])				->       Optional[Any]:

				"""simple docstring"""

				self.assertEqual(self.tokenizer.vocab_size       ,			1_0000)

	def 							_SCREAMING_SNAKE_CASE   (     self:		Dict)				->       int:

				"""simple docstring"""

				self.assertIn(_SCREAMING_SNAKE_CASE       ,			self.tokenizer.all_special_ids)
				__lowerCAmelCase      :  Any												=       [ES_CODE, 4, 1601, 47, 7647, 2]
				__lowerCAmelCase      :  str												=       self.tokenizer.decode(_SCREAMING_SNAKE_CASE       ,			skip_special_tokens=_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase      :  Tuple												=       self.tokenizer.decode(generated_ids[1:]       ,			skip_special_tokens=_SCREAMING_SNAKE_CASE)
				self.assertEqual(_SCREAMING_SNAKE_CASE       ,			_SCREAMING_SNAKE_CASE)
				self.assertNotIn(self.tokenizer.eos_token       ,			_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE   (     self:		Optional[int])				->       int:

				"""simple docstring"""

				__lowerCAmelCase      :  Tuple												=       "fr"
				__lowerCAmelCase      :  Optional[int]												=       self.tokenizer(self.french_text).input_ids
				self.assertEqual(encoded[0]       ,			_SCREAMING_SNAKE_CASE)
				self.assertEqual(encoded[-1]       ,			self.tokenizer.eos_token_id)







	def 							_SCREAMING_SNAKE_CASE   (     self:		List[str])				->       Union[str, Any]:

				"""simple docstring"""

				__lowerCAmelCase      :  Union[str, Any]												=       "fr"
				self.assertListEqual(self.tokenizer.prefix_tokens       ,			[FR_CODE])

				__lowerCAmelCase      :  Tuple												=       "es"
				self.assertListEqual(self.tokenizer.prefix_tokens       ,			[ES_CODE])




"""simple docstring"""






from typing import TYPE_CHECKING

from ..utils import _LazyModule


__snake_case							:   Optional[int]      =    {
    'config': [
        'EXTERNAL_DATA_FORMAT_SIZE_LIMIT',
        'OnnxConfig',
        'OnnxConfigWithPast',
        'OnnxSeq2SeqConfigWithPast',
        'PatchingSpec',
    ],
    'convert': ['export', 'validate_model_outputs'],
    'features': ['FeaturesManager'],
    'utils': ['ParameterFormat', 'compute_serialized_parameters_size'],
}


if TYPE_CHECKING:
	from .config import (
	    EXTERNAL_DATA_FORMAT_SIZE_LIMIT,
	    OnnxConfig,
	    OnnxConfigWithPast,
	    OnnxSeqaSeqConfigWithPast,
	    PatchingSpec,
	)
	from .convert import export, validate_model_outputs
	from .features import FeaturesManager
	from .utils import ParameterFormat, compute_serialized_parameters_size

else:
	import sys

	__snake_case							:   Any      =    _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)