





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_mbart import MBartTokenizer
else:
    snake_case_				=			None


snake_case_				=			logging.get_logger(__name__)


snake_case_				=			{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

snake_case_				=			{
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-en-ro': 'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json',
        'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json',
    },
}

snake_case_				=			{
    'facebook/mbart-large-en-ro': 1024,
    'facebook/mbart-large-cc25': 1024,
}

# fmt: off
snake_case_				=			['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']

class 		SCREAMING_SNAKE_CASE__				(       _UpperCAmelCase  ):
      A_						:				Dict							=    VOCAB_FILES_NAMES
      A_						:				Optional[int]							=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      A_						:				str							=    PRETRAINED_VOCAB_FILES_MAP
      A_						:				int							=    ['input_ids', 'attention_mask']
      A_						:				Tuple							=    MBartTokenizer

      A_						:				List[int]							=    []
      A_						:				List[int]							=    []


      def __init__(self       :      Optional[int]				,			a__       :      str=None				,			a__       :      int=None				,			a__       :      Any="<s>"				,			a__       :      Any="</s>"				,			a__       :      Any="</s>"				,			a__       :      List[str]="<s>"				,			a__       :      List[str]="<unk>"				,			a__       :      Dict="<pad>"				,			a__       :      List[Any]="<mask>"				,			a__       :      Union[str, Any]=None				,			a__       :      Any=None				,			a__       :      Union[str, Any]=None				,			**a__       :      int				,			):





         """simple docstring"""



         __snake_case							       =   AddedToken(a__				,			lstrip=a__				,			rstrip=a__      ) if isinstance(a__				,			a__      ) else mask_token

         super().__init__(
             vocab_file=a__				,			tokenizer_file=a__				,			bos_token=a__				,			eos_token=a__				,			sep_token=a__				,			cls_token=a__				,			unk_token=a__				,			pad_token=a__				,			mask_token=a__				,			src_lang=a__				,			tgt_lang=a__				,			additional_special_tokens=a__				,			**a__				,			)

         __snake_case							       =   vocab_file
         __snake_case							       =   False if not self.vocab_file else True

         __snake_case							       =   FAIRSEQ_LANGUAGE_CODES.copy()

         if additional_special_tokens is not None:
            # Only add those special tokens if they are not already there.
            _additional_special_tokens.extend(
                [t for t in additional_special_tokens if t not in _additional_special_tokens]      )

         self.add_special_tokens({'''additional_special_tokens''': _additional_special_tokens}      )
         __snake_case							       =   {
             lang_code: self.convert_tokens_to_ids(a__      ) for lang_code in FAIRSEQ_LANGUAGE_CODES
         }

         __snake_case							       =   src_lang if src_lang is not None else '''en_XX'''
         __snake_case							       =   self.convert_tokens_to_ids(self._src_lang      )
         __snake_case							       =   tgt_lang
         self.set_src_lang_special_tokens(self._src_lang      )


      @property
      def    a (self       :      Optional[int]      ):





         """simple docstring"""



         return self._src_lang


      @src_lang.setter
      def    a (self       :      Tuple				,			a__       :      str      ):





         """simple docstring"""



         __snake_case							       =   new_src_lang
         self.set_src_lang_special_tokens(self._src_lang      )


      def    a (self       :      Tuple				,			a__       :      List[int]				,			a__       :      Optional[List[int]] = None      ):





         """simple docstring"""



         if token_ids_a is None:
            return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


      def    a (self       :      List[Any]				,			a__       :      List[int]				,			a__       :      Optional[List[int]] = None      ):





         """simple docstring"""



         __snake_case							       =   [self.sep_token_id]
         __snake_case							       =   [self.cls_token_id]

         if token_ids_a is None:
            return len(cls + token_ids_a + sep      ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]


      def    a (self       :      List[Any]				,			a__       :      Optional[Any]				,			a__       :      str				,			a__       :      Optional[str]				,			a__       :      Optional[str]				,			**a__       :      int      ):





         """simple docstring"""



         if src_lang is None or tgt_lang is None:
            raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''      )
         __snake_case							       =   src_lang
         __snake_case							       =   self(a__				,			add_special_tokens=a__				,			return_tensors=a__				,			**a__      )
         __snake_case							       =   self.convert_tokens_to_ids(a__      )
         __snake_case							       =   tgt_lang_id
         return inputs


      def    a (self       :      Tuple				,			a__       :      List[str]				,			a__       :      str = "en_XX"				,			a__       :      Optional[List[str]] = None				,			a__       :      str = "ro_RO"				,			**a__       :      Tuple				,			):





         """simple docstring"""



         __snake_case							       =   src_lang
         __snake_case							       =   tgt_lang
         return super().prepare_seqaseq_batch(a__				,			a__				,			**a__      )


      def    a (self       :      str      ):





         """simple docstring"""



         return self.set_src_lang_special_tokens(self.src_lang      )


      def    a (self       :      List[Any]      ):





         """simple docstring"""



         return self.set_tgt_lang_special_tokens(self.tgt_lang      )


      def    a (self       :      Optional[int]				,			a__       :      str      ):





         """simple docstring"""



         __snake_case							       =   self.convert_tokens_to_ids(a__      )
         __snake_case							       =   []
         __snake_case							       =   [self.eos_token_id, self.cur_lang_code]

         __snake_case							       =   self.convert_ids_to_tokens(self.prefix_tokens      )
         __snake_case							       =   self.convert_ids_to_tokens(self.suffix_tokens      )

         __snake_case							       =   processors.TemplateProcessing(
             single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,			pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,			self.prefix_tokens + self.suffix_tokens      )      )				,			)


      def    a (self       :      Union[str, Any]				,			a__       :      str      ):





         """simple docstring"""



         __snake_case							       =   self.convert_tokens_to_ids(a__      )
         __snake_case							       =   []
         __snake_case							       =   [self.eos_token_id, self.cur_lang_code]

         __snake_case							       =   self.convert_ids_to_tokens(self.prefix_tokens      )
         __snake_case							       =   self.convert_ids_to_tokens(self.suffix_tokens      )

         __snake_case							       =   processors.TemplateProcessing(
             single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,			pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,			self.prefix_tokens + self.suffix_tokens      )      )				,			)


      def    a (self       :      int				,			a__       :      str				,			a__       :      Optional[str] = None      ):





         """simple docstring"""



         if not self.can_save_slow_tokenizer:
            raise ValueError(
                '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                '''tokenizer.'''      )

         if not os.path.isdir(a__      ):
            logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""      )
            return
         __snake_case							       =   os.path.join(
             a__				,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

         if os.path.abspath(self.vocab_file      ) != os.path.abspath(a__      ):
            copyfile(self.vocab_file				,			a__      )

         return (out_vocab_file,)





import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
	    OpenAIGPTConfig,
	    OpenAIGPTDoubleHeadsModel,
	    OpenAIGPTForSequenceClassification,
	    OpenAIGPTLMHeadModel,
	    OpenAIGPTModel,
	)





class       A    :


							def __init__(self      ,				lowerCAmelCase      ,				lowerCAmelCase=1_3      ,				lowerCAmelCase=7      ,				lowerCAmelCase=True      ,				lowerCAmelCase=True      ,				lowerCAmelCase=True      ,				lowerCAmelCase=9_9      ,				lowerCAmelCase=3_2      ,				lowerCAmelCase=5      ,				lowerCAmelCase=4      ,				lowerCAmelCase=3_7      ,				lowerCAmelCase="gelu"      ,				lowerCAmelCase=0.1      ,				lowerCAmelCase=0.1      ,				lowerCAmelCase=5_1_2      ,				lowerCAmelCase=1_6      ,				lowerCAmelCase=2      ,				lowerCAmelCase=0.02      ,				lowerCAmelCase=3      ,				lowerCAmelCase=4      ,				lowerCAmelCase=None      ,				):
									__lowercase=			parent
									__lowercase=			batch_size
									__lowercase=			seq_length
									__lowercase=			is_training
									__lowercase=			use_token_type_ids
									__lowercase=			use_labels
									__lowercase=			vocab_size
									__lowercase=			hidden_size
									__lowercase=			num_hidden_layers
									__lowercase=			num_attention_heads
									__lowercase=			intermediate_size
									__lowercase=			hidden_act
									__lowercase=			hidden_dropout_prob
									__lowercase=			attention_probs_dropout_prob
									__lowercase=			max_position_embeddings
									__lowercase=			type_vocab_size
									__lowercase=			type_sequence_label_size
									__lowercase=			initializer_range
									__lowercase=			num_labels
									__lowercase=			num_choices
									__lowercase=			scope
									__lowercase=			self.vocab_size - 1


							def 			_A       (self					):
									__lowercase=			ids_tensor([self.batch_size, self.seq_length]      ,				self.vocab_size					)

									__lowercase=			None
									if self.use_token_type_ids:
											__lowercase=			ids_tensor([self.batch_size, self.seq_length]      ,				self.type_vocab_size					)

									__lowercase=			None
									__lowercase=			None
									__lowercase=			None
									if self.use_labels:
											__lowercase=			ids_tensor([self.batch_size]      ,				self.type_sequence_label_size					)
											__lowercase=			ids_tensor([self.batch_size, self.seq_length]      ,				self.num_labels					)
											__lowercase=			ids_tensor([self.batch_size]      ,				self.num_choices					)

									__lowercase=			OpenAIGPTConfig(
									    vocab_size=self.vocab_size      ,				n_embd=self.hidden_size      ,				n_layer=self.num_hidden_layers      ,				n_head=self.num_attention_heads      ,				n_positions=self.max_position_embeddings      ,				pad_token_id=self.pad_token_id      ,				)

									__lowercase=			ids_tensor([self.num_hidden_layers, self.num_attention_heads]      ,				2					)

									return (
									    config,
									    input_ids,
									    head_mask,
									    token_type_ids,
									    sequence_labels,
									    token_labels,
									    choice_labels,
									)


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				*lowerCAmelCase					):
									__lowercase=			OpenAIGPTModel(config=lowerCAmelCase					)
									model.to(lowerCAmelCase					)
									model.eval()

									__lowercase=			model(lowerCAmelCase      ,				token_type_ids=lowerCAmelCase      ,				head_mask=lowerCAmelCase					)
									__lowercase=			model(lowerCAmelCase      ,				token_type_ids=lowerCAmelCase					)
									__lowercase=			model(lowerCAmelCase					)

									self.parent.assertEqual(result.last_hidden_state.shape      ,				(self.batch_size, self.seq_length, self.hidden_size)					)


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				*lowerCAmelCase					):
									__lowercase=			OpenAIGPTLMHeadModel(lowerCAmelCase					)
									model.to(lowerCAmelCase					)
									model.eval()

									__lowercase=			model(lowerCAmelCase      ,				token_type_ids=lowerCAmelCase      ,				labels=lowerCAmelCase					)
									self.parent.assertEqual(result.loss.shape      ,				()					)
									self.parent.assertEqual(result.logits.shape      ,				(self.batch_size, self.seq_length, self.vocab_size)					)


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				*lowerCAmelCase					):
									__lowercase=			OpenAIGPTDoubleHeadsModel(lowerCAmelCase					)
									model.to(lowerCAmelCase					)
									model.eval()

									__lowercase=			model(lowerCAmelCase      ,				token_type_ids=lowerCAmelCase      ,				labels=lowerCAmelCase					)
									self.parent.assertEqual(result.loss.shape      ,				()					)
									self.parent.assertEqual(result.logits.shape      ,				(self.batch_size, self.seq_length, self.vocab_size)					)


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				*lowerCAmelCase					):
									__lowercase=			self.num_labels
									__lowercase=			OpenAIGPTForSequenceClassification(lowerCAmelCase					)
									model.to(lowerCAmelCase					)
									model.eval()

									__lowercase=			ids_tensor([self.batch_size]      ,				self.type_sequence_label_size					)
									__lowercase=			model(lowerCAmelCase      ,				token_type_ids=lowerCAmelCase      ,				labels=lowerCAmelCase					)
									self.parent.assertEqual(result.logits.shape      ,				(self.batch_size, self.num_labels)					)







							def 			_A       (self					):
									__lowercase=			self.prepare_config_and_inputs()
									(
									    (
									    __lowercase
									),							(
									    __lowercase
									),							(
									    __lowercase
									),							(
									    __lowercase
									),							(
									    __lowercase
									),							(
									    __lowercase
									),							(
									    __lowercase
									),							
									)=			config_and_inputs
									__lowercase=			{
									    'input_ids': input_ids,
									    'token_type_ids': token_type_ids,
									    'head_mask': head_mask,
									}

									return config, inputs_dict





@require_torch
class       A    (   A_					,				A_					,				A_					,				unittest.TestCase ):
							UpperCamelCase_   :							Optional[Any]       =(
							    (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
							    if is_torch_available()
							    else ()
							)
							UpperCamelCase_   :							Tuple       =(
							    (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
							)  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
							UpperCamelCase_   :							List[str]       =(
							    {
							        '''feature-extraction''': OpenAIGPTModel,
							        '''text-classification''': OpenAIGPTForSequenceClassification,
							        '''text-generation''': OpenAIGPTLMHeadModel,
							        '''zero-shot''': OpenAIGPTForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
											# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
											# `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
											# tiny config could not be created.
											return True

									return False


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase=False					):
									__lowercase=			super()._prepare_for_class(lowerCAmelCase      ,				lowerCAmelCase      ,				return_labels=lowerCAmelCase					)

									if return_labels:
											if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
													__lowercase=			torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)      ,				dtype=torch.long      ,				device=lowerCAmelCase      ,				)
													__lowercase=			inputs_dict['labels']
													__lowercase=			inputs_dict['labels']
													__lowercase=			torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.num_choices)      ,				dtype=torch.long      ,				device=lowerCAmelCase      ,				)
													__lowercase=			torch.zeros(
													    self.model_tester.batch_size      ,				dtype=torch.long      ,				device=lowerCAmelCase					)
									return inputs_dict


							def 			_A       (self					):
									__lowercase=			OpenAIGPTModelTester(self					)
									__lowercase=			ConfigTester(self      ,				config_class=lowerCAmelCase      ,				n_embd=3_7					)


							def 			_A       (self					):
									self.config_tester.run_common_tests()


							def 			_A       (self					):
									__lowercase=			self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_openai_gpt_model(*lowerCAmelCase					)


							def 			_A       (self					):
									__lowercase=			self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_lm_head_model(*lowerCAmelCase					)


							def 			_A       (self					):
									__lowercase=			self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_double_lm_head_model(*lowerCAmelCase					)


							def 			_A       (self					):
									__lowercase=			self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*lowerCAmelCase					)







							@slow
							def 			_A       (self					):
									for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											__lowercase=			OpenAIGPTModel.from_pretrained(lowerCAmelCase					)
											self.assertIsNotNone(lowerCAmelCase					)





@require_torch
class       A    (   unittest.TestCase ):


							@slow
							def 			_A       (self					):
									__lowercase=			OpenAIGPTLMHeadModel.from_pretrained('openai-gpt'					)
									model.to(lowerCAmelCase					)
									__lowercase=			torch.tensor([[4_8_1, 4_7_3_5, 5_4_4]]      ,				dtype=torch.long      ,				device=lowerCAmelCase					)  # the president is
									__lowercase=			[
									    4_8_1,
									    4_7_3_5,
									    5_4_4,
									    2_4_6,
									    9_6_3,
									    8_7_0,
									    7_6_2,
									    2_3_9,
									    2_4_4,
									    4_0_4_7_7,
									    2_4_4,
									    2_4_9,
									    7_1_9,
									    8_8_1,
									    4_8_7,
									    5_4_4,
									    2_4_0,
									    2_4_4,
									    6_0_3,
									    4_8_1,
									]  # the president is a very good man. " \n " i\'m sure he is, " said the

									__lowercase=			model.generate(lowerCAmelCase      ,				do_sample=lowerCAmelCase					)
									self.assertListEqual(output_ids[0].tolist()      ,				lowerCAmelCase					)






