






"""simple docstring"""






import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
  from ...tokenization_utils_base import TextInput
from ...utils import logging


UpperCAmelCase_   : int   			= logging.get_logger(__name__)

UpperCAmelCase_   : Optional[Any]   			= {"""vocab_file""": """spiece.model"""}

UpperCAmelCase_   : Tuple   			= {
    """vocab_file""": {
        """t5-small""": """https://huggingface.co/t5-small/resolve/main/spiece.model""",
        """t5-base""": """https://huggingface.co/t5-base/resolve/main/spiece.model""",
        """t5-large""": """https://huggingface.co/t5-large/resolve/main/spiece.model""",
        """t5-3b""": """https://huggingface.co/t5-3b/resolve/main/spiece.model""",
        """t5-11b""": """https://huggingface.co/t5-11b/resolve/main/spiece.model""",
    }
}


# TODO(PVP) - this should be removed in Transformers v5
UpperCAmelCase_   : List[str]   			= {
    """t5-small""": 512,
    """t5-base""": 512,
    """t5-large""": 512,
    """t5-3b""": 512,
    """t5-11b""": 512,
}

UpperCAmelCase_   : Optional[int]   			= """▁"""




class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






      '''simple docstring'''

      __UpperCamelCase	 =  VOCAB_FILES_NAMES
      __UpperCamelCase	 =  PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase	 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase	 =  ["input_ids", "attention_mask"]
      def __init__(						self     :  List[Any]						,	lowercase_     :  List[str]						,	lowercase_     :  int="</s>"						,	lowercase_     :  Optional[Any]="<unk>"						,	lowercase_     :  List[Any]="<pad>"						,	lowercase_     :  Union[str, Any]=100						,	lowercase_     :  Any=None						,	lowercase_     :  Optional[Dict[str, Any]] = None						,	lowercase_     :  List[str]=True						,	**lowercase_     :  Dict						,	):




          '''simple docstring'''


          if extra_ids > 0 and additional_special_tokens is None:
              SCREAMING_SNAKE_CASE_      :  Dict    						=		[F'<extra_id_{i}>' for i in range(lowercase_)]
          elif extra_ids > 0 and additional_special_tokens is not None:
              # Check that we have the right number of extra_id special tokens
              SCREAMING_SNAKE_CASE_      :  str    						=		len(set(filter(lambda lowercase_: bool('''extra_id''' in str(lowercase_))						,	lowercase_)))
              if extra_tokens != extra_ids:
                  raise ValueError(
                      F'Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'
                      ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
                      ''' tokens''')
          if legacy:
              logger.warning_once(
                  F'You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to'
                  ''' read the related pull request available at https://github.com/huggingface/transformers/pull/24565''')

          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		legacy
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		{} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              eos_token=lowercase_						,	unk_token=lowercase_						,	pad_token=lowercase_						,	extra_ids=lowercase_						,	additional_special_tokens=lowercase_						,	sp_model_kwargs=self.sp_model_kwargs						,	legacy=lowercase_						,	**lowercase_						,	)

          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		vocab_file
          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		extra_ids

          SCREAMING_SNAKE_CASE_      :  int    						=		spm.SentencePieceProcessor(**self.sp_model_kwargs)
          self.sp_model.Load(lowercase_)
      @staticmethod
      def _SCREAMING_SNAKE_CASE (						lowercase_     :  List[str]						,	lowercase_     :  Tuple						,	lowercase_     :  List[str]):




          '''simple docstring'''


          if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
              SCREAMING_SNAKE_CASE_      :  Any    						=		TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
              if init_max_model_length is not None and init_max_model_length != max_model_length:
                  return init_max_model_length
              elif init_max_model_length is None:
                  warnings.warn(
                      '''This tokenizer was incorrectly instantiated with a model max length of'''
                      F' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'
                      ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
                      ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
                      F' {pretrained_model_name_or_path} automatically truncating your input to'
                      F' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'
                      F' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'
                      ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
                      ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''						,	lowercase_						,	)

          return max_model_length
      @property
      def _SCREAMING_SNAKE_CASE (						self     :  str):




          '''simple docstring'''


          return self.sp_model.get_piece_size() + self._extra_ids
      def _SCREAMING_SNAKE_CASE (						self     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		{self.convert_ids_to_tokens(lowercase_): i for i in range(self.vocab_size)}
          vocab.update(self.added_tokens_encoder)
          return vocab
      def _SCREAMING_SNAKE_CASE (						self     :  List[Any]						,	lowercase_     :  List[int]						,	lowercase_     :  Optional[List[int]] = None						,	lowercase_     :  bool = False):




          '''simple docstring'''


          if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=lowercase_						,	token_ids_a=lowercase_						,	already_has_special_tokens=lowercase_)

          # normal case: some special tokens
          if token_ids_a is None:
              return ([0] * len(lowercase_)) + [1]
          return ([0] * len(lowercase_)) + [1] + ([0] * len(lowercase_)) + [1]
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]):




          '''simple docstring'''


          return list(
              set(filter(lambda lowercase_: bool(re.search(r'''<extra_id_\d+>'''						,	lowercase_)) is not None						,	self.additional_special_tokens)))
      def _SCREAMING_SNAKE_CASE (						self     :  Dict):




          '''simple docstring'''


          return [self._convert_token_to_id(lowercase_) for token in self.get_sentinel_tokens()]
      def _SCREAMING_SNAKE_CASE (						self     :  List[Any]						,	lowercase_     :  List[int]):




          '''simple docstring'''


          if len(lowercase_) > 0 and token_ids[-1] == self.eos_token_id:
              warnings.warn(
                  F'This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'
                  ''' eos tokens being added.''')
              return token_ids
          else:
              return token_ids + [self.eos_token_id]
      def _SCREAMING_SNAKE_CASE (						self     :  List[Any]						,	lowercase_     :  List[int]						,	lowercase_     :  Optional[List[int]] = None):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		[self.eos_token_id]

          if token_ids_a is None:
              return len(token_ids_a + eos) * [0]
          return len(token_ids_a + eos + token_ids_a + eos) * [0]
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]						,	lowercase_     :  List[int]						,	lowercase_     :  Optional[List[int]] = None):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  int    						=		self._add_eos_if_not_present(lowercase_)
          if token_ids_a is None:
              return token_ids_a
          else:
              SCREAMING_SNAKE_CASE_      :  Tuple    						=		self._add_eos_if_not_present(lowercase_)
              return token_ids_a + token_ids_a
      def __getstate__(						self     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  int    						=		self.__dict__.copy()
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		None
          return state
      def __setstate__(						self     :  Tuple						,	lowercase_     :  Union[str, Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		d

          # for backward compatibility
          if not hasattr(self						,	'''sp_model_kwargs'''):
              SCREAMING_SNAKE_CASE_      :  Tuple    						=		{}

          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		spm.SentencePieceProcessor(**self.sp_model_kwargs)
          self.sp_model.Load(self.vocab_file)
      def _SCREAMING_SNAKE_CASE (						self     :  Dict						,	lowercase_     :  "TextInput"						,	**lowercase_     :  Optional[Any]):




          '''simple docstring'''


          if not self.legacy:
              SCREAMING_SNAKE_CASE_      :  int    						=		SPIECE_UNDERLINE + text.replace(lowercase_						,	''' ''')
          return super().tokenize(lowercase_						,	**lowercase_)
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]						,	lowercase_     :  Optional[Any]						,	**lowercase_     :  List[Any]):




          '''simple docstring'''


          if not self.legacy:
              SCREAMING_SNAKE_CASE_      :  List[Any]    						=		text.startswith(lowercase_)
              if is_first:
                  SCREAMING_SNAKE_CASE_      :  List[str]    						=		text[1:]

          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		self.sp_model.encode(lowercase_						,	out_type=lowercase_)

          if not self.legacy and not is_first and not text.startswith(''' ''') and tokens[0].startswith(lowercase_):
              SCREAMING_SNAKE_CASE_      :  Any    						=		([tokens[0][1:]] if len(tokens[0]) > 1 else []) + tokens[1:]
          return tokens
      def _SCREAMING_SNAKE_CASE (						self     :  List[str]						,	lowercase_     :  Tuple):




          '''simple docstring'''


          if token.startswith('''<extra_id_'''):
              SCREAMING_SNAKE_CASE_      :  int    						=		re.match(r'''<extra_id_(\d+)>'''						,	lowercase_)
              SCREAMING_SNAKE_CASE_      :  List[str]    						=		int(match.group(1))
              return self.vocab_size - num - 1
          return self.sp_model.piece_to_id(lowercase_)
      def _SCREAMING_SNAKE_CASE (						self     :  int						,	lowercase_     :  List[str]):




          '''simple docstring'''


          if index < self.sp_model.get_piece_size():
              SCREAMING_SNAKE_CASE_      :  Any    						=		self.sp_model.IdToPiece(lowercase_)
          else:
              SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		F'<extra_id_{self.vocab_size - 1 - index}>'
          return token
      def _SCREAMING_SNAKE_CASE (						self     :  Dict						,	lowercase_     :  Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Tuple    						=		[]
          SCREAMING_SNAKE_CASE_      :  int    						=		''''''
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		False
          for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                  if not prev_is_special:
                      out_string += " "
                  out_string += self.sp_model.decode(lowercase_) + token
                  SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		True
                  SCREAMING_SNAKE_CASE_      :  str    						=		[]
              else:
                  current_sub_tokens.append(lowercase_)
                  SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		False
          out_string += self.sp_model.decode(lowercase_)
          return out_string.strip()



      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]						,	lowercase_     :  str						,	lowercase_     :  Optional[str] = None):




          '''simple docstring'''


          if not os.path.isdir(lowercase_):
              logger.error(F'Vocabulary path ({save_directory}) should be a directory')
              return
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		os.path.join(
              lowercase_						,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

          if os.path.abspath(self.vocab_file) != os.path.abspath(lowercase_) and os.path.isfile(self.vocab_file):
              copyfile(self.vocab_file						,	lowercase_)
          elif not os.path.isfile(self.vocab_file):
              with open(lowercase_						,	'''wb''') as fi:
                  SCREAMING_SNAKE_CASE_      :  Any    						=		self.sp_model.serialized_model_proto()
                  fi.write(lowercase_)

          return (out_vocab_file,)










"""simple docstring"""






import warnings

from ...utils import logging
from .image_processing_deit import DeiTImageProcessor


UpperCAmelCase_   : Optional[Any]   			= logging.get_logger(__name__)




class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






      '''simple docstring'''

      def __init__(						self     :  List[str]						,	*lowercase_     :  Dict						,	**lowercase_     :  Union[str, Any]):




          '''simple docstring'''


          warnings.warn(
              '''The class DeiTFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please'''
              ''' use DeiTImageProcessor instead.'''						,	lowercase_						,	)
          super().__init__(*lowercase_						,	**lowercase_)


