




"""simple docstring"""






import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
			from ...tokenization_utils_base import TextInput
from ...utils import logging


lowercase__			=		logging.get_logger(__name__)

lowercase__			=		{"""vocab_file""": """spiece.model"""}

lowercase__			=		{
    """vocab_file""": {
        """t5-small""": """https://huggingface.co/t5-small/resolve/main/spiece.model""",
        """t5-base""": """https://huggingface.co/t5-base/resolve/main/spiece.model""",
        """t5-large""": """https://huggingface.co/t5-large/resolve/main/spiece.model""",
        """t5-3b""": """https://huggingface.co/t5-3b/resolve/main/spiece.model""",
        """t5-11b""": """https://huggingface.co/t5-11b/resolve/main/spiece.model""",
    }
}


# TODO(PVP) - this should be removed in Transformers v5
lowercase__			=		{
    """t5-small""": 512,
    """t5-base""": 512,
    """t5-large""": 512,
    """t5-3b""": 512,
    """t5-11b""": 512,
}

lowercase__			=		"""▁"""






class lowerCAmelCase__      (	lowercase		):






			'''simple docstring'''



			lowerCamelCase__ =     VOCAB_FILES_NAMES
			lowerCamelCase__ =     PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__ =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__ =     ["""input_ids""", """attention_mask"""]
			def __init__(    self    , lowercase    , lowercase="</s>"    , lowercase="<unk>"    , lowercase="<pad>"    , lowercase=100    , lowercase=None    , lowercase = None    , lowercase=True    , **lowercase    , ):
							# Add extra_ids to the special token list
							if extra_ids > 0 and additional_special_tokens is None:
											_lowerCamelCase	:     List[str]       =       [F'''<extra_id_{i}>''' for i in range(lowercase    )]
							elif extra_ids > 0 and additional_special_tokens is not None:
											# Check that we have the right number of extra_id special tokens
											_lowerCamelCase	:     Any       =       len(set(filter(lambda lowercase    : bool('extra_id' in str(lowercase    )    )    , lowercase    )    )    )
											if extra_tokens != extra_ids:
															raise ValueError(
															    F'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
															    ' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'
															    ' tokens'    )
							if legacy:
											logger.warning_once(
											    F'''You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to'''
											    ' read the related pull request available at https://github.com/huggingface/transformers/pull/24565'    )

							_lowerCamelCase	:     int       =       legacy
							_lowerCamelCase	:     Tuple       =       {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    eos_token=lowercase    , unk_token=lowercase    , pad_token=lowercase    , extra_ids=lowercase    , additional_special_tokens=lowercase    , sp_model_kwargs=self.sp_model_kwargs    , legacy=lowercase    , **lowercase    , )

							_lowerCamelCase	:     Optional[Any]       =       vocab_file
							_lowerCamelCase	:     Optional[int]       =       extra_ids

							_lowerCamelCase	:     List[str]       =       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
							self.sp_model.Load(lowercase    )
			@staticmethod
			def    A_       (    lowercase    , lowercase    , lowercase    ):
							if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
											_lowerCamelCase	:     Optional[Any]       =       TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
											if init_max_model_length is not None and init_max_model_length != max_model_length:
															return init_max_model_length
											elif init_max_model_length is None:
															warnings.warn(
															    'This tokenizer was incorrectly instantiated with a model max length of'
															    F''' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'''
															    ' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'
															    ' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'
															    F''' {pretrained_model_name_or_path} automatically truncating your input to'''
															    F''' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'''
															    F''' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'''
															    ' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'
															    ' instantiate this tokenizer with `model_max_length` set to your preferred value.'    , lowercase    , )

							return max_model_length
			@property
			def    A_       (    self    ):
							return self.sp_model.get_piece_size() + self._extra_ids
			def    A_       (    self    ):
							_lowerCamelCase	:     str       =       {self.convert_ids_to_tokens(lowercase    ): i for i in range(self.vocab_size    )}
							vocab.update(self.added_tokens_encoder    )
							return vocab
			def    A_       (    self    , lowercase    , lowercase = None    , lowercase = False    ):
							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=lowercase    , token_ids_a=lowercase    , already_has_special_tokens=lowercase    )

							# normal case: some special tokens
							if token_ids_a is None:
											return ([0] * len(lowercase    )) + [1]
							return ([0] * len(lowercase    )) + [1] + ([0] * len(lowercase    )) + [1]
			def    A_       (    self    ):
							return list(
							    set(filter(lambda lowercase    : bool(re.search(r'<extra_id_\d+>'    , lowercase    )    ) is not None    , self.additional_special_tokens    )    )    )
			def    A_       (    self    ):
							return [self._convert_token_to_id(lowercase    ) for token in self.get_sentinel_tokens()]
			def    A_       (    self    , lowercase    ):
							if len(lowercase    ) > 0 and token_ids[-1] == self.eos_token_id:
											warnings.warn(
											    F'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
											    ' eos tokens being added.'    )
											return token_ids
							else:
											return token_ids + [self.eos_token_id]
			def    A_       (    self    , lowercase    , lowercase = None    ):
							_lowerCamelCase	:     int       =       [self.eos_token_id]

							if token_ids_a is None:
											return len(token_ids_a + eos    ) * [0]
							return len(token_ids_a + eos + token_ids_a + eos    ) * [0]
			def    A_       (    self    , lowercase    , lowercase = None    ):
							_lowerCamelCase	:     int       =       self._add_eos_if_not_present(lowercase    )
							if token_ids_a is None:
											return token_ids_a
							else:
											_lowerCamelCase	:     Any       =       self._add_eos_if_not_present(lowercase    )
											return token_ids_a + token_ids_a
			def __getstate__(    self    ):
							_lowerCamelCase	:     Dict       =       self.__dict__.copy()
							_lowerCamelCase	:     Dict       =       None
							return state
			def __setstate__(    self    , lowercase    ):
							_lowerCamelCase	:     str       =       d

							# for backward compatibility
							if not hasattr(self    , 'sp_model_kwargs'    ):
											_lowerCamelCase	:     Union[str, Any]       =       {}

							_lowerCamelCase	:     List[Any]       =       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
							self.sp_model.Load(self.vocab_file    )
			def    A_       (    self    , lowercase    , **lowercase    ):
							# Replace the SPIECE_UNDERLINE with a space to make sure SPIECE_UNDERLINE is only used at
							# the beginning of the text
							if not self.legacy:
											_lowerCamelCase	:     List[Any]       =       SPIECE_UNDERLINE + text.replace(lowercase    , ' '    )
							return super().tokenize(lowercase    , **lowercase    )
			def    A_       (    self    , lowercase    , **lowercase    ):
							if not self.legacy:
											_lowerCamelCase	:     Optional[int]       =       text.startswith(lowercase    )
											if is_first:
															_lowerCamelCase	:     List[Any]       =       text[1:]

							_lowerCamelCase	:     int       =       self.sp_model.encode(lowercase    , out_type=lowercase    )

							if not self.legacy and not is_first and not text.startswith(' '    ) and tokens[0].startswith(lowercase    ):
											_lowerCamelCase	:     List[Any]       =       ([tokens[0][1:]] if len(tokens[0]    ) > 1 else []) + tokens[1:]
							return tokens
			def    A_       (    self    , lowercase    ):
							if token.startswith('<extra_id_'    ):
											_lowerCamelCase	:     int       =       re.match(r'<extra_id_(\d+)>'    , lowercase    )
											_lowerCamelCase	:     Dict       =       int(match.group(1    )    )
											return self.vocab_size - num - 1
							return self.sp_model.piece_to_id(lowercase    )
			def    A_       (    self    , lowercase    ):
							if index < self.sp_model.get_piece_size():
											_lowerCamelCase	:     Optional[Any]       =       self.sp_model.IdToPiece(lowercase    )
							else:
											_lowerCamelCase	:     Dict       =       F'''<extra_id_{self.vocab_size - 1 - index}>'''
							return token
			def    A_       (    self    , lowercase    ):
							_lowerCamelCase	:     List[Any]       =       []
							_lowerCamelCase	:     Dict       =       ''
							_lowerCamelCase	:     Tuple       =       False
							for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
															if not prev_is_special:
																			out_string += " "
															out_string += self.sp_model.decode(lowercase    ) + token
															_lowerCamelCase	:     Tuple       =       True
															_lowerCamelCase	:     int       =       []
											else:
															current_sub_tokens.append(lowercase    )
															_lowerCamelCase	:     Any       =       False
							out_string += self.sp_model.decode(lowercase    )
							return out_string.strip()






			def    A_       (    self    , lowercase    , lowercase = None    ):
							if not os.path.isdir(lowercase    ):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
											return
							_lowerCamelCase	:     str       =       os.path.join(
							    lowercase    , (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

							if os.path.abspath(self.vocab_file    ) != os.path.abspath(lowercase    ) and os.path.isfile(self.vocab_file    ):
											copyfile(self.vocab_file    , lowercase    )
							elif not os.path.isfile(self.vocab_file    ):
											with open(lowercase    , 'wb'    ) as fi:
															_lowerCamelCase	:     Optional[Any]       =       self.sp_model.serialized_model_proto()
															fi.write(lowercase    )

							return (out_vocab_file,)





"""simple docstring"""






import unittest

from transformers import MPNetConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    MPNetForMaskedLM,
			    MPNetForMultipleChoice,
			    MPNetForQuestionAnswering,
			    MPNetForSequenceClassification,
			    MPNetForTokenClassification,
			    MPNetModel,
			)






class lowerCAmelCase__      :






			'''simple docstring'''



			def __init__(    self    , lowercase    , lowercase=13    , lowercase=7    , lowercase=True    , lowercase=True    , lowercase=False    , lowercase=True    , lowercase=99    , lowercase=64    , lowercase=5    , lowercase=4    , lowercase=64    , lowercase="gelu"    , lowercase=0.1    , lowercase=0.1    , lowercase=512    , lowercase=16    , lowercase=2    , lowercase=0.02    , lowercase=3    , lowercase=4    , lowercase=None    , ):
							_lowerCamelCase	:     List[str]       =       parent
							_lowerCamelCase	:     List[str]       =       batch_size
							_lowerCamelCase	:     Tuple       =       seq_length
							_lowerCamelCase	:     List[Any]       =       is_training
							_lowerCamelCase	:     int       =       use_input_mask
							_lowerCamelCase	:     Optional[Any]       =       use_token_type_ids
							_lowerCamelCase	:     List[Any]       =       use_labels
							_lowerCamelCase	:     str       =       vocab_size
							_lowerCamelCase	:     Optional[int]       =       hidden_size
							_lowerCamelCase	:     List[str]       =       num_hidden_layers
							_lowerCamelCase	:     Optional[int]       =       num_attention_heads
							_lowerCamelCase	:     List[str]       =       intermediate_size
							_lowerCamelCase	:     int       =       hidden_act
							_lowerCamelCase	:     Optional[int]       =       hidden_dropout_prob
							_lowerCamelCase	:     List[Any]       =       attention_probs_dropout_prob
							_lowerCamelCase	:     List[str]       =       max_position_embeddings
							_lowerCamelCase	:     List[Any]       =       type_vocab_size
							_lowerCamelCase	:     Optional[int]       =       type_sequence_label_size
							_lowerCamelCase	:     Union[str, Any]       =       initializer_range
							_lowerCamelCase	:     Tuple       =       num_labels
							_lowerCamelCase	:     int       =       num_choices
							_lowerCamelCase	:     int       =       scope
			def    A_       (    self    ):
							return MPNetConfig.from_pretrained('microsoft/mpnet-base'    )
			def    A_       (    self    ):
							_lowerCamelCase	:     Tuple       =       ids_tensor([self.batch_size, self.seq_length]    , self.vocab_size    )

							_lowerCamelCase	:     Dict       =       None
							if self.use_input_mask:
											_lowerCamelCase	:     Optional[int]       =       random_attention_mask([self.batch_size, self.seq_length]    )

							_lowerCamelCase	:     int       =       None
							_lowerCamelCase	:     List[str]       =       None
							_lowerCamelCase	:     Any       =       None
							if self.use_labels:
											_lowerCamelCase	:     List[str]       =       ids_tensor([self.batch_size]    , self.type_sequence_label_size    )
											_lowerCamelCase	:     str       =       ids_tensor([self.batch_size, self.seq_length]    , self.num_labels    )
											_lowerCamelCase	:     Dict       =       ids_tensor([self.batch_size]    , self.num_choices    )

							_lowerCamelCase	:     int       =       self.get_config()
							return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
			def    A_       (    self    ):
							return MPNetConfig(
							    vocab_size=self.vocab_size    , hidden_size=self.hidden_size    , num_hidden_layers=self.num_hidden_layers    , num_attention_heads=self.num_attention_heads    , intermediate_size=self.intermediate_size    , hidden_act=self.hidden_act    , hidden_dropout_prob=self.hidden_dropout_prob    , attention_probs_dropout_prob=self.attention_probs_dropout_prob    , max_position_embeddings=self.max_position_embeddings    , initializer_range=self.initializer_range    , )
			def    A_       (    self    , lowercase    , lowercase    , lowercase    , lowercase    , lowercase    , lowercase    ):
							_lowerCamelCase	:     Optional[Any]       =       MPNetModel(config=lowercase    )
							model.to(lowercase    )
							model.eval()
							_lowerCamelCase	:     Tuple       =       model(lowercase    , lowercase    )
							_lowerCamelCase	:     Tuple       =       model(lowercase    )
							self.parent.assertEqual(result.last_hidden_state.shape    , (self.batch_size, self.seq_length, self.hidden_size)    )
							self.parent.assertEqual(result.pooler_output.shape    , (self.batch_size, self.hidden_size)    )
			def    A_       (    self    , lowercase    , lowercase    , lowercase    , lowercase    , lowercase    , lowercase    ):
							_lowerCamelCase	:     str       =       MPNetForQuestionAnswering(config=lowercase    )
							model.to(lowercase    )
							model.eval()
							_lowerCamelCase	:     str       =       model(
							    lowercase    , attention_mask=lowercase    , start_positions=lowercase    , end_positions=lowercase    , )
							self.parent.assertEqual(result.start_logits.shape    , (self.batch_size, self.seq_length)    )
							self.parent.assertEqual(result.end_logits.shape    , (self.batch_size, self.seq_length)    )
			def    A_       (    self    , lowercase    , lowercase    , lowercase    , lowercase    , lowercase    , lowercase    ):
							_lowerCamelCase	:     Optional[int]       =       self.num_labels
							_lowerCamelCase	:     int       =       MPNetForSequenceClassification(lowercase    )
							model.to(lowercase    )
							model.eval()
							_lowerCamelCase	:     Optional[Any]       =       model(lowercase    , attention_mask=lowercase    , labels=lowercase    )
							self.parent.assertEqual(result.logits.shape    , (self.batch_size, self.num_labels)    )
			def    A_       (    self    , lowercase    , lowercase    , lowercase    , lowercase    , lowercase    , lowercase    ):
							_lowerCamelCase	:     List[str]       =       self.num_choices
							_lowerCamelCase	:     List[str]       =       MPNetForMultipleChoice(config=lowercase    )
							model.to(lowercase    )
							model.eval()
							_lowerCamelCase	:     Union[str, Any]       =       input_ids.unsqueeze(1    ).expand(-1    , self.num_choices    , -1    ).contiguous()
							_lowerCamelCase	:     List[str]       =       input_mask.unsqueeze(1    ).expand(-1    , self.num_choices    , -1    ).contiguous()
							_lowerCamelCase	:     Any       =       model(
							    lowercase    , attention_mask=lowercase    , labels=lowercase    , )
							self.parent.assertEqual(result.logits.shape    , (self.batch_size, self.num_choices)    )
			def    A_       (    self    , lowercase    , lowercase    , lowercase    , lowercase    , lowercase    , lowercase    ):
							_lowerCamelCase	:     Any       =       self.num_labels
							_lowerCamelCase	:     Union[str, Any]       =       MPNetForTokenClassification(config=lowercase    )
							model.to(lowercase    )
							model.eval()
							_lowerCamelCase	:     Union[str, Any]       =       model(lowercase    , attention_mask=lowercase    , labels=lowercase    )
							self.parent.assertEqual(result.logits.shape    , (self.batch_size, self.seq_length, self.num_labels)    )






			def    A_       (    self    ):
							_lowerCamelCase	:     Dict       =       self.prepare_config_and_inputs()
							((_lowerCamelCase),					(_lowerCamelCase),					(_lowerCamelCase),					(_lowerCamelCase),					(_lowerCamelCase),					(_lowerCamelCase))	:     List[Any]       =       config_and_inputs
							_lowerCamelCase	:     str       =       {'input_ids': input_ids, 'attention_mask': input_mask}
							return config, inputs_dict






@require_torch
class lowerCAmelCase__      (	lowercase,      lowercase,      unittest.TestCase		):






			'''simple docstring'''



			lowerCamelCase__ =     (
			    (
			        MPNetForMaskedLM,
			        MPNetForMultipleChoice,
			        MPNetForQuestionAnswering,
			        MPNetForSequenceClassification,
			        MPNetForTokenClassification,
			        MPNetModel,
			    )
			    if is_torch_available()
			    else ()
			)
			lowerCamelCase__ =     (
			    {
			        """feature-extraction""": MPNetModel,
			        """fill-mask""": MPNetForMaskedLM,
			        """question-answering""": MPNetForQuestionAnswering,
			        """text-classification""": MPNetForSequenceClassification,
			        """token-classification""": MPNetForTokenClassification,
			        """zero-shot""": MPNetForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			lowerCamelCase__ =     False
			lowerCamelCase__ =     True
			def    A_       (    self    ):
							_lowerCamelCase	:     Union[str, Any]       =       MPNetModelTester(self    )
							_lowerCamelCase	:     List[Any]       =       ConfigTester(self    , config_class=lowercase    , hidden_size=37    )
			def    A_       (    self    ):
							self.config_tester.run_common_tests()
			def    A_       (    self    ):
							_lowerCamelCase	:     Union[str, Any]       =       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mpnet_model(*lowercase    )
			def    A_       (    self    ):
							_lowerCamelCase	:     List[Any]       =       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mpnet_for_sequence_classification(*lowercase    )
			def    A_       (    self    ):
							_lowerCamelCase	:     Optional[Any]       =       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mpnet_for_multiple_choice(*lowercase    )
			def    A_       (    self    ):
							_lowerCamelCase	:     List[Any]       =       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mpnet_for_token_classification(*lowercase    )






			def    A_       (    self    ):
							_lowerCamelCase	:     str       =       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mpnet_for_question_answering(*lowercase    )






@require_torch
class lowerCAmelCase__      (	unittest.TestCase		):






			'''simple docstring'''



			@slow
			def    A_       (    self    ):
							_lowerCamelCase	:     Optional[Any]       =       MPNetModel.from_pretrained('microsoft/mpnet-base'    )
							_lowerCamelCase	:     Optional[Any]       =       torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]    )
							_lowerCamelCase	:     Any       =       model(lowercase    )[0]
							_lowerCamelCase	:     Dict       =       torch.Size((1, 11, 768)    )
							self.assertEqual(output.shape    , lowercase    )
							_lowerCamelCase	:     List[Any]       =       torch.tensor(
							    [[[-0.05_50, 0.19_43, -0.07_40], [-0.05_62, 0.22_11, -0.05_79], [-0.04_37, 0.33_37, -0.06_41]]]    )
							# compare the actual values for a slice.
							self.assertTrue(torch.allclose(output[:, :3, :3]    , lowercase    , atol=1E-4    )    )