
'''simple docstring'''
import copy
import tempfile
import unittest

from transformers import MaMaaaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
from transformers.utils import cached_property

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import MaMaaaForConditionalGeneration, MaMaaaModel, MaMaaaTokenizer
						from transformers.models.mam_aaa.modeling_mam_aaa import MaMaaaDecoder, MaMaaaEncoder




def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__=None ,       SCREAMING_SNAKE_CASE__=None ,       SCREAMING_SNAKE_CASE__=None ,       SCREAMING_SNAKE_CASE__=None ,       SCREAMING_SNAKE_CASE__=None ,       )  ->							Optional[int]:


		'''simple docstring'''


		if attention_mask is None:
				snake_case  :       Dict     			=						input_ids.ne(config.pad_token_id			)
		if decoder_attention_mask is None:
				snake_case  :       List[Any]     			=						decoder_input_ids.ne(config.pad_token_id			)
		if head_mask is None:
				snake_case  :       Dict     			=						torch.ones(config.encoder_layers ,       config.encoder_attention_heads ,       device=SCREAMING_SNAKE_CASE__			)
		if decoder_head_mask is None:
				snake_case  :       Optional[Any]     			=						torch.ones(config.decoder_layers ,       config.decoder_attention_heads ,       device=SCREAMING_SNAKE_CASE__			)
		if cross_attn_head_mask is None:
				snake_case  :       Tuple     			=						torch.ones(config.decoder_layers ,       config.decoder_attention_heads ,       device=SCREAMING_SNAKE_CASE__			)
		return {
		    "input_ids": input_ids,
		    "decoder_input_ids": decoder_input_ids,
		    "attention_mask": attention_mask,
		    "decoder_attention_mask": attention_mask,
		    "head_mask": head_mask,
		    "decoder_head_mask": decoder_head_mask,
		    "cross_attn_head_mask": cross_attn_head_mask,
		}






class 	snake_case__       :







		"""simple docstring"""

		def __init__(      self				:			Optional[int]  ,	UpperCamelCase__				:			Any  ,	UpperCamelCase__				:			List[Any]=13  ,	UpperCamelCase__				:			Optional[int]=7  ,	UpperCamelCase__				:			Tuple=True  ,	UpperCamelCase__				:			Dict=False  ,	UpperCamelCase__				:			int=99  ,	UpperCamelCase__				:			Union[str, Any]=16  ,	UpperCamelCase__				:			Optional[int]=2  ,	UpperCamelCase__				:			Union[str, Any]=4  ,	UpperCamelCase__				:			Union[str, Any]=4  ,	UpperCamelCase__				:			str="relu"  ,	UpperCamelCase__				:			Optional[int]=0.1  ,	UpperCamelCase__				:			Tuple=0.1  ,	UpperCamelCase__				:			List[Any]=0.0  ,	UpperCamelCase__				:			List[Any]=0.0  ,	UpperCamelCase__				:			Tuple=20  ,	UpperCamelCase__				:			Any=2  ,	UpperCamelCase__				:			Union[str, Any]=1  ,	UpperCamelCase__				:			List[Any]=0  ,	)						->			Optional[Any]:






				"""simple docstring"""







				snake_case  :       Union[str, Any]     			=						parent
				snake_case  :       List[Any]     			=						batch_size
				snake_case  :       int     			=						seq_length
				snake_case  :       int     			=						is_training
				snake_case  :       List[Any]     			=						use_labels
				snake_case  :       List[Any]     			=						vocab_size
				snake_case  :       Optional[Any]     			=						hidden_size
				snake_case  :       str     			=						num_hidden_layers
				snake_case  :       Optional[Any]     			=						num_attention_heads
				snake_case  :       List[Any]     			=						intermediate_size
				snake_case  :       str     			=						hidden_act
				snake_case  :       Optional[int]     			=						hidden_dropout_prob
				snake_case  :       Optional[int]     			=						attention_probs_dropout_prob
				snake_case  :       Dict     			=						encoder_layerdrop
				snake_case  :       Optional[Any]     			=						decoder_layerdrop
				snake_case  :       Dict     			=						max_position_embeddings
				snake_case  :       List[str]     			=						eos_token_id
				snake_case  :       Dict     			=						pad_token_id
				snake_case  :       int     			=						bos_token_id



		def       lowerCAmelCase				(      self				:			Optional[Any]      )						->			str:






				"""simple docstring"""







				snake_case  :       Any     			=						ids_tensor([self.batch_size, self.seq_length]  ,	self.vocab_size      )
				snake_case  :       List[Any]     			=						self.eos_token_id  # Eos Token
				snake_case  :       int     			=						ids_tensor([self.batch_size, self.seq_length]  ,	self.vocab_size      )

				# we need to clamp the input ids here to avoid having pad token in between
				# this is because for M2M100 the position_ids are prepared such that
				# all pad tokens have pos id = 2 and rest are between 2..seq_length
				# and the seq_length here is seq_length - num_pad_tokens
				# but when using past, there is no way of knowing if the past input ids had
				# pad tokens in them, which results in incorrect seq_lenth and which in turn results in
				# position_ids being off by num_pad_tokens in past input
				snake_case  :       str     			=						input_ids.clamp(self.pad_token_id + 1      )
				snake_case  :       Optional[Any]     			=						decoder_input_ids.clamp(self.pad_token_id + 1      )

				snake_case  :       Union[str, Any]     			=						self.get_config()
				snake_case  :       Tuple     			=						prepare_mam_aaa_inputs_dict(UpperCamelCase__  ,	UpperCamelCase__  ,	UpperCamelCase__      )
				return config, inputs_dict



		def       lowerCAmelCase				(      self				:			int      )						->			Dict:






				"""simple docstring"""







				return MaMaaaConfig(
				    vocab_size=self.vocab_size  ,	d_model=self.hidden_size  ,	encoder_layers=self.num_hidden_layers  ,	decoder_layers=self.num_hidden_layers  ,	encoder_attention_heads=self.num_attention_heads  ,	decoder_attention_heads=self.num_attention_heads  ,	encoder_ffn_dim=self.intermediate_size  ,	decoder_ffn_dim=self.intermediate_size  ,	dropout=self.hidden_dropout_prob  ,	attention_dropout=self.attention_probs_dropout_prob  ,	encoder_layerdrop=self.encoder_layerdrop  ,	decoder_layerdrop=self.decoder_layerdrop  ,	max_position_embeddings=self.max_position_embeddings  ,	eos_token_id=self.eos_token_id  ,	bos_token_id=self.bos_token_id  ,	pad_token_id=self.pad_token_id  ,	)



		def       lowerCAmelCase				(      self				:			Optional[Any]      )						->			List[Any]:






				"""simple docstring"""







				snake_case			,snake_case  :       str     			=						self.prepare_config_and_inputs()
				return config, inputs_dict



		def       lowerCAmelCase				(      self				:			Union[str, Any]  ,	UpperCamelCase__				:			Union[str, Any]  ,	UpperCamelCase__				:			str      )						->			Union[str, Any]:






				"""simple docstring"""







				snake_case  :       Tuple     			=						MaMaaaModel(config=UpperCamelCase__      ).get_decoder().to(UpperCamelCase__      ).eval()
				snake_case  :       Dict     			=						inputs_dict['''input_ids''']
				snake_case  :       Union[str, Any]     			=						inputs_dict['''attention_mask''']
				snake_case  :       List[str]     			=						inputs_dict['''head_mask''']

				# first forward pass
				snake_case  :       Any     			=						model(UpperCamelCase__  ,	attention_mask=UpperCamelCase__  ,	head_mask=UpperCamelCase__  ,	use_cache=UpperCamelCase__      )

				snake_case			,snake_case  :       List[str]     			=						outputs.to_tuple()

				# create hypothetical multiple next token and extent to next_input_ids
				snake_case  :       List[Any]     			=						ids_tensor((self.batch_size, 3)  ,	config.vocab_size      )
				snake_case  :       List[Any]     			=						ids_tensor((self.batch_size, 3)  ,	2      )

				# append to next input_ids and
				snake_case  :       Tuple     			=						torch.cat([input_ids, next_tokens]  ,	dim=-1      )
				snake_case  :       List[str]     			=						torch.cat([attention_mask, next_attn_mask]  ,	dim=-1      )

				snake_case  :       Tuple     			=						model(UpperCamelCase__  ,	attention_mask=UpperCamelCase__      )['''last_hidden_state''']
				snake_case  :       Any     			=						model(UpperCamelCase__  ,	attention_mask=UpperCamelCase__  ,	past_key_values=UpperCamelCase__      )[
				    '''last_hidden_state'''
				]

				# select random slice
				snake_case  :       Dict     			=						ids_tensor((1,)  ,	output_from_past.shape[-1]      ).item()
				snake_case  :       Any     			=						output_from_no_past[:, -3:, random_slice_idx].detach()
				snake_case  :       List[str]     			=						output_from_past[:, :, random_slice_idx].detach()

				self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]      )

				# test that outputs are equal for slice
				self.parent.assertTrue(torch.allclose(UpperCamelCase__  ,	UpperCamelCase__  ,	atol=1e-2      )      )



		def       lowerCAmelCase				(      self				:			str  ,	UpperCamelCase__				:			List[Any]  ,	UpperCamelCase__				:			Dict      )						->			Any:






				"""simple docstring"""







				snake_case  :       Optional[int]     			=						MaMaaaModel(config=UpperCamelCase__      ).to(UpperCamelCase__      ).eval()
				snake_case  :       Optional[int]     			=						model(**UpperCamelCase__      )

				snake_case  :       Dict     			=						outputs.encoder_last_hidden_state
				snake_case  :       List[str]     			=						outputs.last_hidden_state

				with tempfile.TemporaryDirectory() as tmpdirname:
						snake_case  :       List[str]     			=						model.get_encoder()
						encoder.save_pretrained(UpperCamelCase__      )
						snake_case  :       int     			=						MaMaaaEncoder.from_pretrained(UpperCamelCase__      ).to(UpperCamelCase__      )

				snake_case  :       Dict     			=						encoder(inputs_dict['''input_ids''']  ,	attention_mask=inputs_dict['''attention_mask''']      )[
				    0
				]

				self.parent.assertTrue((encoder_last_hidden_state_a - encoder_last_hidden_state).abs().max().item() < 1e-3      )

				with tempfile.TemporaryDirectory() as tmpdirname:
						snake_case  :       Optional[int]     			=						model.get_decoder()
						decoder.save_pretrained(UpperCamelCase__      )
						snake_case  :       Tuple     			=						MaMaaaDecoder.from_pretrained(UpperCamelCase__      ).to(UpperCamelCase__      )

				snake_case  :       str     			=						decoder(
				    input_ids=inputs_dict['''decoder_input_ids''']  ,	attention_mask=inputs_dict['''decoder_attention_mask''']  ,	encoder_hidden_states=UpperCamelCase__  ,	encoder_attention_mask=inputs_dict['''attention_mask''']  ,	)[0]

				self.parent.assertTrue((last_hidden_state_a - last_hidden_state).abs().max().item() < 1e-3      )







@require_torch
class 	snake_case__       (      __SCREAMING_SNAKE_CASE    ,					__SCREAMING_SNAKE_CASE    ,					__SCREAMING_SNAKE_CASE    ,					unittest.TestCase					):







		"""simple docstring"""

		lowerCamelCase     							=						(
		    (
		        MaMaaaModel,
		        MaMaaaForConditionalGeneration,
		    )
		    if is_torch_available()
		    else ()
		)
		lowerCamelCase     							=						(MaMaaaForConditionalGeneration,) if is_torch_available() else ()
		lowerCamelCase     							=						(
		    {
		        """conversational""": MaMaaaForConditionalGeneration,
		        """feature-extraction""": MaMaaaModel,
		        """summarization""": MaMaaaForConditionalGeneration,
		        """text2text-generation""": MaMaaaForConditionalGeneration,
		        """translation""": MaMaaaForConditionalGeneration,
		    }
		    if is_torch_available()
		    else {}
		)
		lowerCamelCase     							=						True
		lowerCamelCase     							=						True
		lowerCamelCase     							=						False
		lowerCamelCase     							=						False



		def       lowerCAmelCase				(      self				:			List[Any]  ,	UpperCamelCase__				:			Tuple  ,	UpperCamelCase__				:			List[Any]  ,	UpperCamelCase__				:			Dict  ,	UpperCamelCase__				:			List[Any]  ,	UpperCamelCase__				:			int      )						->			Optional[Any]:






				"""simple docstring"""







				if pipeline_test_casse_name == "TranslationPipelineTests":
						# Get `ValueError: Translation requires a `src_lang` and a `tgt_lang` for this model`.
						# `M2M100Config` was never used in pipeline tests: cannot create a simple tokenizer.
						return True

				return False



		def       lowerCAmelCase				(      self				:			Tuple      )						->			List[str]:






				"""simple docstring"""







				snake_case  :       Any     			=						MaMaaaModelTester(self      )
				snake_case  :       List[str]     			=						ConfigTester(self  ,	config_class=UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			Optional[int]      )						->			str:






				"""simple docstring"""







				self.config_tester.run_common_tests()



		def       lowerCAmelCase				(      self				:			List[str]      )						->			int:






				"""simple docstring"""







				snake_case			,snake_case  :       Union[str, Any]     			=						self.model_tester.prepare_config_and_inputs()
				for model_class in self.all_model_classes:
						snake_case  :       int     			=						model_class(UpperCamelCase__      )

						with tempfile.TemporaryDirectory() as tmpdirname:
								model.save_pretrained(UpperCamelCase__      )
								snake_case			,snake_case  :       List[str]     			=						model_class.from_pretrained(UpperCamelCase__  ,	output_loading_info=UpperCamelCase__      )
						self.assertEqual(info['''missing_keys''']  ,	[]      )



		def       lowerCAmelCase				(      self				:			int      )						->			str:






				"""simple docstring"""







				snake_case  :       Optional[Any]     			=						self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_decoder_model_past_large_inputs(*UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			Optional[Any]      )						->			Any:






				"""simple docstring"""







				snake_case  :       Optional[Any]     			=						self.model_tester.prepare_config_and_inputs_for_common()
				self.model_tester.check_encoder_decoder_model_standalone(*UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			Optional[Any]      )						->			Any:






				"""simple docstring"""







				snake_case			,snake_case  :       Union[str, Any]     			=						self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in (MaMaaaModel, MaMaaaForConditionalGeneration):
						snake_case  :       List[str]     			=						model_class(UpperCamelCase__      )
						model.to(UpperCamelCase__      )
						model.eval()

						snake_case  :       Optional[int]     			=						copy.deepcopy(self._prepare_for_class(UpperCamelCase__  ,	UpperCamelCase__      )      )

						if not self.is_encoder_decoder:
								snake_case  :       str     			=						inputs['''input_ids''']
								del inputs["input_ids"]
						else:
								snake_case  :       Any     			=						inputs['''input_ids''']
								snake_case  :       Any     			=						inputs.get('''decoder_input_ids'''  ,	UpperCamelCase__      )
								del inputs["input_ids"]
								inputs.pop('''decoder_input_ids'''  ,	UpperCamelCase__      )

						snake_case  :       int     			=						model.get_input_embeddings()
						if not self.is_encoder_decoder:
								snake_case  :       int     			=						wte(UpperCamelCase__      )
						else:
								snake_case  :       int     			=						wte(UpperCamelCase__      )
								snake_case  :       List[Any]     			=						wte(UpperCamelCase__      )

						with torch.no_grad():
								model(**UpperCamelCase__      )[0]



		def       lowerCAmelCase				(      self				:			int      )						->			Any:






				"""simple docstring"""







				snake_case			,snake_case  :       int     			=						self.model_tester.prepare_config_and_inputs()
				snake_case  :       List[Any]     			=						input_dict['''input_ids''']
				snake_case  :       int     			=						input_ids.ne(1      ).to(UpperCamelCase__      )
				snake_case  :       Tuple     			=						MaMaaaForConditionalGeneration(UpperCamelCase__      ).eval().to(UpperCamelCase__      )
				if torch_device == "cuda":
						model.half()
				model.generate(UpperCamelCase__  ,	attention_mask=UpperCamelCase__      )
				model.generate(num_beams=4  ,	do_sample=UpperCamelCase__  ,	early_stopping=UpperCamelCase__  ,	num_return_sequences=3      )







def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__			)  ->							List[Any]:


		'''simple docstring'''


		return torch.tensor(SCREAMING_SNAKE_CASE__ ,       dtype=torch.long ,       device=SCREAMING_SNAKE_CASE__			)


lowercase__						       =      1e-4






@require_torch
@require_sentencepiece
@require_tokenizers
@slow
class 	snake_case__       (      unittest.TestCase					):







		"""simple docstring"""

		@cached_property
		def       lowerCAmelCase				(      self				:			List[str]      )						->			Any:






				"""simple docstring"""







				return MaMaaaTokenizer.from_pretrained('''facebook/m2m100_418M'''      )



		def       lowerCAmelCase				(      self				:			Tuple      )						->			Dict:






				"""simple docstring"""







				snake_case  :       Dict     			=						MaMaaaModel.from_pretrained('''facebook/m2m100_418M'''      ).to(UpperCamelCase__      )
				snake_case  :       List[str]     			=						_long_tensor([[12_8028, 98, 12, 3_0527, 2732, 159, 7755, 6_1904, 3_9144, 38, 2]]      )
				snake_case  :       str     			=						_long_tensor([[2, 12_8028, 98, 12, 3_0527, 2732, 159, 7755, 6_1904, 3_9144, 38]]      )
				snake_case  :       Optional[int]     			=						prepare_mam_aaa_inputs_dict(model.config  ,	UpperCamelCase__  ,	UpperCamelCase__      )
				with torch.no_grad():
						snake_case  :       Dict     			=						model(**UpperCamelCase__      )[0]
				snake_case  :       str     			=						torch.Size((1, 11, 1024)      )
				self.assertEqual(output.shape  ,	UpperCamelCase__      )
				# change to expected output here
				snake_case  :       Tuple     			=						torch.tensor(
				    [[-0.7_780, -0.1_676, 0.1_038], [-6.7_556, -1.3_992, 0.0_567], [-7.5_383, -0.5_920, -0.2_779]]  ,	device=UpperCamelCase__      )
				self.assertTrue(torch.allclose(output[:, :3, :3]  ,	UpperCamelCase__  ,	atol=UpperCamelCase__      )      )



		def       lowerCAmelCase				(      self				:			List[Any]      )						->			Optional[Any]:






				"""simple docstring"""







				snake_case  :       Tuple     			=						MaMaaaForConditionalGeneration.from_pretrained('''facebook/m2m100_418M'''      ).to(UpperCamelCase__      )

				# change to intended input
				snake_case  :       Tuple     			=						_long_tensor([[12_8028, 98, 12, 3_0527, 2732, 159, 7755, 6_1904, 3_9144, 38, 2]]      )
				snake_case  :       Union[str, Any]     			=						_long_tensor([[2, 12_8028, 98, 12, 3_0527, 2732, 159, 7755, 6_1904, 3_9144, 38]]      )
				snake_case  :       Dict     			=						prepare_mam_aaa_inputs_dict(model.config  ,	UpperCamelCase__  ,	UpperCamelCase__      )
				with torch.no_grad():
						snake_case  :       Tuple     			=						model(**UpperCamelCase__      )[0]
				snake_case  :       Any     			=						torch.Size((1, 11, model.config.vocab_size)      )
				self.assertEqual(output.shape  ,	UpperCamelCase__      )
				# change to expected output here
				snake_case  :       Optional[int]     			=						torch.tensor(
				    [[-1.0_448, -1.0_411, 3.7_992], [-3.2_191, -3.2_386, -1.3_451], [-3.6_210, -3.5_993, 0.4_925]]  ,	device=UpperCamelCase__      )
				self.assertTrue(torch.allclose(output[:, :3, :3]  ,	UpperCamelCase__  ,	atol=UpperCamelCase__      )      )



		def       lowerCAmelCase				(      self				:			Tuple      )						->			str:






				"""simple docstring"""







				snake_case  :       Dict     			=						MaMaaaForConditionalGeneration.from_pretrained('''facebook/m2m100_418M'''      ).to(UpperCamelCase__      )
				snake_case  :       Optional[int]     			=						MaMaaaTokenizer.from_pretrained('''facebook/m2m100_418M'''  ,	src_lang='''fr'''  ,	tgt_lang='''en'''      )

				snake_case  :       List[Any]     			=						[
				    '''L\'affaire NSA souligne l\'absence totale de débat sur le renseignement''',
				    '''Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.''',
				    '''Lorsque François Hollande téléphone à Barack Obama ou quand le ministre des affaires étrangères Laurent'''
				    ''' Fabius convoque l\'ambassadeur des Etats-Unis, ils réagissent à une vraie découverte, qui est celle de'''
				    ''' l\'ampleur de la surveillance américaine sur l\'ensemble des communications en France.''',
				]

				# The below article tests that we don't add any hypotheses outside of the top n_beams
				snake_case  :       Optional[int]     			=						tokenizer(UpperCamelCase__  ,	padding=UpperCamelCase__  ,	return_tensors='''pt'''      )

				snake_case  :       Tuple     			=						model.generate(
				    input_ids=dct['''input_ids'''].to(UpperCamelCase__      )  ,	attention_mask=dct['''attention_mask'''].to(UpperCamelCase__      )  ,	num_beams=5  ,	forced_bos_token_id=tokenizer.get_lang_id('''en'''      )  ,	)

				snake_case  :       Dict     			=						[
				    '''The NSA case highlights the total absence of intelligence debate''',
				    '''I think there are two levels of response from the French government.''',
				    '''When François Hollande calls Barack Obama or when Foreign Minister Laurent Fabius calls the U.S.'''
				    ''' Ambassador, they respond to a real discovery, which is that of the scale of U.S. surveillance on all'''
				    ''' communications in France.''',
				]

				snake_case  :       List[str]     			=						tokenizer.batch_decode(
				    hypotheses_batch.tolist()  ,	clean_up_tokenization_spaces=UpperCamelCase__  ,	skip_special_tokens=UpperCamelCase__      )
				assert generated == expected_en




'''simple docstring'''
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
						from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
						lowercase__						       =      get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
						from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowercase__						       =      1_2_8_0_2_2
lowercase__						       =      1_2_8_0_2_8






@require_sentencepiece
class 	snake_case__       (      __SCREAMING_SNAKE_CASE    ,					unittest.TestCase					):







		"""simple docstring"""

		lowerCamelCase     							=						MaMaaaTokenizer
		lowerCamelCase     							=						False
		lowerCamelCase     							=						False
		lowerCamelCase     							=						True



		def       lowerCAmelCase				(      self				:			Dict      )						->			Optional[Any]:






				"""simple docstring"""







				super().setUp()

				snake_case  :       int     			=						['''</s>''', '''<unk>''', '''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''', '''\u0120''', '''<pad>''']
				snake_case  :       int     			=						dict(zip(UpperCamelCase__  ,	range(len(UpperCamelCase__      )      )      )      )
				snake_case  :       Optional[int]     			=						Path(self.tmpdirname      )
				save_json(UpperCamelCase__  ,	save_dir / VOCAB_FILES_NAMES['''vocab_file''']      )
				if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
						copyfile(UpperCamelCase__  ,	save_dir / VOCAB_FILES_NAMES['''spm_file''']      )

				snake_case  :       Optional[int]     			=						MaMaaaTokenizer.from_pretrained(self.tmpdirname      )
				tokenizer.save_pretrained(self.tmpdirname      )



		def       lowerCAmelCase				(      self				:			Dict  ,	**UpperCamelCase__				:			Optional[int]      )						->			Any:






				"""simple docstring"""







				return MaMaaaTokenizer.from_pretrained(self.tmpdirname  ,	**UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			Tuple  ,	UpperCamelCase__				:			Tuple      )						->			str:






				"""simple docstring"""







				return (
				    "This is a test",
				    "This is a test",
				)



		def       lowerCAmelCase				(      self				:			List[str]      )						->			Optional[Any]:






				"""simple docstring"""







				snake_case  :       Optional[int]     			=						'''</s>'''
				snake_case  :       str     			=						0

				self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__      )  ,	UpperCamelCase__      )
				self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__      )  ,	UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			Dict      )						->			int:






				"""simple docstring"""







				snake_case  :       Tuple     			=						self.get_tokenizer()
				snake_case  :       Dict     			=						list(tokenizer.get_vocab().keys()      )

				self.assertEqual(vocab_keys[0]  ,	'''</s>'''      )
				self.assertEqual(vocab_keys[1]  ,	'''<unk>'''      )
				self.assertEqual(vocab_keys[-1]  ,	'''<s>'''      )
				self.assertEqual(len(UpperCamelCase__      )  ,	tokenizer.vocab_size + len(tokenizer.get_added_vocab()      )      )



		@unittest.skip('''Skip this test while all models are still to be uploaded.'''      )
		def       lowerCAmelCase				(      self				:			str      )						->			List[str]:






				"""simple docstring"""







				pass



		def       lowerCAmelCase				(      self				:			int      )						->			Union[str, Any]:






				"""simple docstring"""







				snake_case  :       Optional[Any]     			=						self.get_tokenizer()

				snake_case  :       Optional[Any]     			=						tokenizer.tokenize('''This is a test'''      )
				self.assertListEqual(UpperCamelCase__  ,	['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

				self.assertListEqual(
				    tokenizer.convert_tokens_to_ids(UpperCamelCase__      )  ,	[2, 3, 4, 5, 6]  ,	)

				snake_case  :       Any     			=						tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6]      )
				self.assertListEqual(UpperCamelCase__  ,	['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

				snake_case  :       Optional[int]     			=						tokenizer.convert_tokens_to_string(UpperCamelCase__      )
				self.assertEqual(UpperCamelCase__  ,	'''This is a test'''      )



		@slow
		def       lowerCAmelCase				(      self				:			Union[str, Any]      )						->			List[Any]:






				"""simple docstring"""







				snake_case  :       Tuple     			=						{'''input_ids''': [[12_8022, 11_0108, 397, 11, 3_8272, 2247, 12_4811, 285, 1_8105, 1586, 207, 7, 3_9534, 4428, 397, 1019, 1_8105, 1586, 207, 7, 4_1337, 1_6786, 241, 7, 2_0214, 17, 12_5690, 1_0398, 7, 4_4378, 5_8069, 6_8342, 7798, 7343, 11, 299, 3_3310, 4, 158, 3_7350, 9_4077, 4569, 299, 3_3310, 90, 4, 5_2840, 290, 4, 3_1270, 112, 299, 682, 4, 5_2840, 3_9953, 1_4079, 193, 5_2519, 9_0894, 1_7894, 12_0697, 11, 4_0445, 551, 17, 1019, 5_2519, 9_0894, 1_7756, 963, 11, 4_0445, 480, 17, 9792, 1120, 5173, 1393, 6240, 1_6786, 241, 12_0996, 28, 1245, 1393, 11_8240, 1_1123, 1019, 9_3612, 2691, 1_0618, 9_8058, 12_0409, 1928, 279, 4, 4_0683, 367, 178, 207, 1019, 103, 10_3121, 506, 6_5296, 5, 2], [12_8022, 2_1217, 367, 117, 12_5450, 128, 719, 7, 7308, 40, 9_3612, 1_2669, 1116, 1_6704, 71, 1_7785, 3699, 1_5592, 35, 144, 9584, 241, 1_1943, 713, 950, 799, 2247, 8_8427, 150, 149, 11_8813, 12_0706, 1019, 10_6906, 8_1518, 28, 1224, 2_2799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [12_8022, 1658, 12_3311, 5155, 5578, 4722, 279, 1_4947, 2366, 1120, 1197, 14, 1348, 9232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
				# fmt: on

				self.tokenizer_integration_test_util(
				    expected_encoding=UpperCamelCase__  ,	model_name='''facebook/m2m100_418M'''  ,	revision='''c168bae485c864188cf9aa0e4108b0b6934dc91e'''  ,	)







@require_torch
@require_sentencepiece
@require_tokenizers
class 	snake_case__       (      unittest.TestCase					):







		"""simple docstring"""

		lowerCamelCase     							=						"""facebook/m2m100_418M"""
		lowerCamelCase     							=						[
		    """In my opinion, there are two levels of response from the French government.""",
		    """NSA Affair Emphasizes Complete Lack of Debate on Intelligence""",
		]
		lowerCamelCase     							=						[
		    """Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.""",
		    """L'affaire NSA souligne l'absence totale de débat sur le renseignement""",
		]

		# fmt: off
		lowerCamelCase     							=						[EN_CODE, 593, 1949, 115781, 4, 71586, 4234, 60633, 126233, 432, 123808, 15592, 1197, 117132, 120618, 5, 2]



		@classmethod
		def       lowerCAmelCase				(      cls				:			List[Any]      )						->			int:






				"""simple docstring"""







				snake_case  :       MaMaaaTokenizer     			=						MaMaaaTokenizer.from_pretrained(
				    cls.checkpoint_name  ,	src_lang='''en'''  ,	tgt_lang='''fr'''      )
				snake_case  :       List[str]     			=						1
				return cls



		def       lowerCAmelCase				(      self				:			Union[str, Any]      )						->			str:






				"""simple docstring"""







				self.assertEqual(self.tokenizer.get_lang_id('''ar'''      )  ,	12_8006      )
				self.assertEqual(self.tokenizer.get_lang_id('''en'''      )  ,	12_8022      )
				self.assertEqual(self.tokenizer.get_lang_id('''ro'''      )  ,	12_8076      )
				self.assertEqual(self.tokenizer.get_lang_id('''mr'''      )  ,	12_8063      )



		def       lowerCAmelCase				(      self				:			Any      )						->			Dict:






				"""simple docstring"""







				snake_case  :       List[str]     			=						self.tokenizer.get_vocab()
				self.assertEqual(len(UpperCamelCase__      )  ,	self.tokenizer.vocab_size      )
				self.assertEqual(vocab['''<unk>''']  ,	3      )
				self.assertIn(self.tokenizer.get_lang_token('''en'''      )  ,	UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			Optional[Any]      )						->			List[Any]:






				"""simple docstring"""







				snake_case  :       Dict     			=						'''en'''
				snake_case  :       Dict     			=						self.tokenizer.batch_encode_plus(self.src_text      ).input_ids[0]
				self.assertListEqual(self.expected_src_tokens  ,	UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			List[str]      )						->			List[Any]:






				"""simple docstring"""







				self.assertIn(UpperCamelCase__  ,	self.tokenizer.all_special_ids      )
				# fmt: off
				snake_case  :       str     			=						[FR_CODE, 5364, 82, 8642, 4, 294, 47, 8, 1_4028, 136, 3286, 9706, 6, 9_0797, 6, 14_4012, 162, 8_8128, 3_0061, 5, 2]
				# fmt: on
				snake_case  :       int     			=						self.tokenizer.decode(UpperCamelCase__  ,	skip_special_tokens=UpperCamelCase__      )
				snake_case  :       List[Any]     			=						self.tokenizer.decode(generated_ids[1:]  ,	skip_special_tokens=UpperCamelCase__      )
				self.assertEqual(UpperCamelCase__  ,	UpperCamelCase__      )
				self.assertNotIn(self.tokenizer.eos_token  ,	UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			Optional[int]      )						->			Tuple:






				"""simple docstring"""







				snake_case  :       Union[str, Any]     			=						tempfile.mkdtemp()
				snake_case  :       Union[str, Any]     			=						self.tokenizer.lang_token_to_id
				self.tokenizer.save_pretrained(UpperCamelCase__      )
				snake_case  :       Optional[int]     			=						MaMaaaTokenizer.from_pretrained(UpperCamelCase__      )
				self.assertDictEqual(new_tok.lang_token_to_id  ,	UpperCamelCase__      )



		@require_torch
		def       lowerCAmelCase				(      self				:			Optional[Any]      )						->			Union[str, Any]:






				"""simple docstring"""







				snake_case  :       Optional[Any]     			=						'''en'''
				snake_case  :       int     			=						'''fr'''

				snake_case  :       Dict     			=						self.tokenizer(self.src_text  ,	text_target=self.tgt_text  ,	padding=UpperCamelCase__  ,	return_tensors='''pt'''      )

				snake_case  :       List[str]     			=						shift_tokens_right(
				    batch['''labels''']  ,	self.tokenizer.pad_token_id  ,	self.tokenizer.eos_token_id      )

				for k in batch:
						snake_case  :       Tuple     			=						batch[k].tolist()
				# batch = {k: v.tolist() for k,v in batch.items()}
				# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
				# batch.decoder_inputs_ids[0][0] ==
				assert batch.input_ids[1][0] == EN_CODE
				assert batch.input_ids[1][-1] == 2
				assert batch.labels[1][0] == FR_CODE
				assert batch.labels[1][-1] == 2
				assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]



		@require_torch
		def       lowerCAmelCase				(      self				:			int      )						->			Union[str, Any]:






				"""simple docstring"""







				snake_case  :       List[str]     			=						'''mr'''
				self.assertListEqual(self.tokenizer.prefix_tokens  ,	[self.tokenizer.get_lang_id('''mr'''      )]      )
				self.assertListEqual(self.tokenizer.suffix_tokens  ,	[self.tokenizer.eos_token_id]      )

				snake_case  :       Optional[int]     			=						'''zh'''
				self.assertListEqual(self.tokenizer.prefix_tokens  ,	[self.tokenizer.get_lang_id('''zh'''      )]      )
				self.assertListEqual(self.tokenizer.suffix_tokens  ,	[self.tokenizer.eos_token_id]      )



		@require_torch
		def       lowerCAmelCase				(      self				:			Union[str, Any]      )						->			Dict:






				"""simple docstring"""







				snake_case  :       List[Any]     			=						'''mr'''
				self.tokenizer._switch_to_target_mode()
				self.assertListEqual(self.tokenizer.prefix_tokens  ,	[self.tokenizer.get_lang_id('''mr'''      )]      )
				self.assertListEqual(self.tokenizer.suffix_tokens  ,	[self.tokenizer.eos_token_id]      )
				self.tokenizer._switch_to_input_mode()
				self.assertListEqual(self.tokenizer.prefix_tokens  ,	[self.tokenizer.get_lang_id(self.tokenizer.src_lang      )]      )

				snake_case  :       List[str]     			=						'''zh'''
				self.tokenizer._switch_to_target_mode()
				self.assertListEqual(self.tokenizer.prefix_tokens  ,	[self.tokenizer.get_lang_id('''zh'''      )]      )
				self.assertListEqual(self.tokenizer.suffix_tokens  ,	[self.tokenizer.eos_token_id]      )
				self.tokenizer._switch_to_input_mode()
				self.assertListEqual(self.tokenizer.prefix_tokens  ,	[self.tokenizer.get_lang_id(self.tokenizer.src_lang      )]      )



		@require_torch
		def       lowerCAmelCase				(      self				:			Tuple      )						->			str:






				"""simple docstring"""







				snake_case  :       int     			=						self.tokenizer._build_translation_inputs('''A test'''  ,	return_tensors='''pt'''  ,	src_lang='''en'''  ,	tgt_lang='''ar'''      )

				self.assertEqual(
				    nested_simplify(UpperCamelCase__      )  ,	{
				        # en_XX, A, test, EOS
				        '''input_ids''': [[12_8022, 58, 4183, 2]],
				        '''attention_mask''': [[1, 1, 1, 1]],
				        # ar_AR
				        '''forced_bos_token_id''': 12_8006,
				    }  ,	)


