

'''simple docstring'''






import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
 from ...tokenization_utils_base import TextInput
from ...utils import logging


lowerCAmelCase__											=	logging.get_logger(__name__)

lowerCAmelCase__											=	{'''vocab_file''': '''spiece.model'''}

lowerCAmelCase__											=	{
    '''vocab_file''': {
        '''t5-small''': '''https://huggingface.co/t5-small/resolve/main/spiece.model''',
        '''t5-base''': '''https://huggingface.co/t5-base/resolve/main/spiece.model''',
        '''t5-large''': '''https://huggingface.co/t5-large/resolve/main/spiece.model''',
        '''t5-3b''': '''https://huggingface.co/t5-3b/resolve/main/spiece.model''',
        '''t5-11b''': '''https://huggingface.co/t5-11b/resolve/main/spiece.model''',
    }
}


# TODO(PVP) - this should be removed in Transformers v5
lowerCAmelCase__											=	{
    '''t5-small''': 512,
    '''t5-base''': 512,
    '''t5-large''': 512,
    '''t5-3b''': 512,
    '''t5-11b''': 512,
}

lowerCAmelCase__											=	'''▁'''






class 			lowercase_	(lowerCamelCase__      ):




  """simple docstring"""




  SCREAMING_SNAKE_CASE    :  List[str]							=			VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE    :  Optional[Any]							=			PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE    :  Dict							=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE    :  Optional[Any]							=			['input_ids', 'attention_mask']






  def __init__(      self   :     Any					,lowercase__   :     Any					,lowercase__   :     Tuple="</s>"					,lowercase__   :     Union[str, Any]="<unk>"					,lowercase__   :     Union[str, Any]="<pad>"					,lowercase__   :     int=1_0_0					,lowercase__   :     List[Any]=None					,lowercase__   :     Optional[Dict[str, Any]] = None					,lowercase__   :     int=True					,**lowercase__   :     int					,):
   # Add extra_ids to the special token list
   if extra_ids > 0 and additional_special_tokens is None:
    __lowercase							  = [F"<extra_id_{i}>" for i in range(lowercase__			)]
   elif extra_ids > 0 and additional_special_tokens is not None:
    # Check that we have the right number of extra_id special tokens
    __lowercase							  = len(set(filter(lambda lowercase__			: bool('''extra_id''' in str(lowercase__			)			)					,lowercase__			)			)			)
    if extra_tokens != extra_ids:
     raise ValueError(
         F"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
         ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
         ''' tokens'''			)
   if legacy:
    logger.warning_once(
        F"You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to"
        ''' read the related pull request available at https://github.com/huggingface/transformers/pull/24565'''			)

   __lowercase							  = legacy
   __lowercase							  = {} if sp_model_kwargs is None else sp_model_kwargs

   super().__init__(
       eos_token=lowercase__					,unk_token=lowercase__					,pad_token=lowercase__					,extra_ids=lowercase__					,additional_special_tokens=lowercase__					,sp_model_kwargs=self.sp_model_kwargs					,legacy=lowercase__					,**lowercase__					,)

   __lowercase							  = vocab_file
   __lowercase							  = extra_ids

   __lowercase							  = spm.SentencePieceProcessor(**self.sp_model_kwargs			)
   self.sp_model.Load(lowercase__			)






  @staticmethod
  def 					SCREAMING_SNAKE_CASE      (      lowercase__   :     Dict					,lowercase__   :     Optional[Any]					,lowercase__   :     Any			):
   if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
    __lowercase							  = TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
    if init_max_model_length is not None and init_max_model_length != max_model_length:
     return init_max_model_length
    elif init_max_model_length is None:
     warnings.warn(
         '''This tokenizer was incorrectly instantiated with a model max length of'''
         F" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"
         ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
         ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
         F" {pretrained_model_name_or_path} automatically truncating your input to"
         F" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"
         F" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"
         ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
         ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''					,lowercase__					,)

   return max_model_length






  @property
  def 					SCREAMING_SNAKE_CASE      (      self   :     List[Any]			):
   return self.sp_model.get_piece_size() + self._extra_ids






  def 					SCREAMING_SNAKE_CASE      (      self   :     Dict			):
   __lowercase							  = {self.convert_ids_to_tokens(lowercase__			): i for i in range(self.vocab_size			)}
   vocab.update(self.added_tokens_encoder			)
   return vocab






  def 					SCREAMING_SNAKE_CASE      (      self   :     Optional[int]					,lowercase__   :     List[int]					,lowercase__   :     Optional[List[int]] = None					,lowercase__   :     bool = False			):
   if already_has_special_tokens:
    return super().get_special_tokens_mask(
        token_ids_a=lowercase__					,token_ids_a=lowercase__					,already_has_special_tokens=lowercase__			)

   # normal case: some special tokens
   if token_ids_a is None:
    return ([0] * len(lowercase__			)) + [1]
   return ([0] * len(lowercase__			)) + [1] + ([0] * len(lowercase__			)) + [1]






  def 					SCREAMING_SNAKE_CASE      (      self   :     List[str]			):
   return list(
       set(filter(lambda lowercase__			: bool(re.search(r'''<extra_id_\d+>'''					,lowercase__			)			) is not None					,self.additional_special_tokens			)			)			)






  def 					SCREAMING_SNAKE_CASE      (      self   :     int			):
   return [self._convert_token_to_id(lowercase__			) for token in self.get_sentinel_tokens()]






  def 					SCREAMING_SNAKE_CASE      (      self   :     List[str]					,lowercase__   :     List[int]			):
   if len(lowercase__			) > 0 and token_ids[-1] == self.eos_token_id:
    warnings.warn(
        F"This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"
        ''' eos tokens being added.'''			)
    return token_ids
   else:
    return token_ids + [self.eos_token_id]






  def 					SCREAMING_SNAKE_CASE      (      self   :     Tuple					,lowercase__   :     List[int]					,lowercase__   :     Optional[List[int]] = None			):
   __lowercase							  = [self.eos_token_id]

   if token_ids_a is None:
    return len(token_ids_a + eos			) * [0]
   return len(token_ids_a + eos + token_ids_a + eos			) * [0]






  def 					SCREAMING_SNAKE_CASE      (      self   :     Optional[int]					,lowercase__   :     List[int]					,lowercase__   :     Optional[List[int]] = None			):
   __lowercase							  = self._add_eos_if_not_present(lowercase__			)
   if token_ids_a is None:
    return token_ids_a
   else:
    __lowercase							  = self._add_eos_if_not_present(lowercase__			)
    return token_ids_a + token_ids_a






  def __getstate__(      self   :     Optional[Any]			):
   __lowercase							  = self.__dict__.copy()
   __lowercase							  = None
   return state






  def __setstate__(      self   :     Optional[int]					,lowercase__   :     int			):
   __lowercase							  = d

   # for backward compatibility
   if not hasattr(self					,'''sp_model_kwargs'''			):
    __lowercase							  = {}

   __lowercase							  = spm.SentencePieceProcessor(**self.sp_model_kwargs			)
   self.sp_model.Load(self.vocab_file			)






  def 					SCREAMING_SNAKE_CASE      (      self   :     int					,lowercase__   :     "TextInput"					,**lowercase__   :     Optional[int]			):
   # Replace the SPIECE_UNDERLINE with a space to make sure SPIECE_UNDERLINE is only used at
   # the beginning of the text
   if not self.legacy:
    __lowercase							  = SPIECE_UNDERLINE + text.replace(lowercase__					,''' '''			)
   return super().tokenize(lowercase__					,**lowercase__			)






  def 					SCREAMING_SNAKE_CASE      (      self   :     List[str]					,lowercase__   :     Optional[int]					,**lowercase__   :     Any			):
   if not self.legacy:
    __lowercase							  = text.startswith(lowercase__			)
    if is_first:
     __lowercase							  = text[1:]

   __lowercase							  = self.sp_model.encode(lowercase__					,out_type=lowercase__			)

   if not self.legacy and not is_first and not text.startswith(''' '''			) and tokens[0].startswith(lowercase__			):
    __lowercase							  = ([tokens[0][1:]] if len(tokens[0]			) > 1 else []) + tokens[1:]
   return tokens






  def 					SCREAMING_SNAKE_CASE      (      self   :     Dict					,lowercase__   :     Dict			):
   if token.startswith('''<extra_id_'''			):
    __lowercase							  = re.match(r'''<extra_id_(\d+)>'''					,lowercase__			)
    __lowercase							  = int(match.group(1			)			)
    return self.vocab_size - num - 1
   return self.sp_model.piece_to_id(lowercase__			)






  def 					SCREAMING_SNAKE_CASE      (      self   :     Dict					,lowercase__   :     Optional[Any]			):
   if index < self.sp_model.get_piece_size():
    __lowercase							  = self.sp_model.IdToPiece(lowercase__			)
   else:
    __lowercase							  = F"<extra_id_{self.vocab_size - 1 - index}>"
   return token






  def 					SCREAMING_SNAKE_CASE      (      self   :     Tuple					,lowercase__   :     int			):
   __lowercase							  = []
   __lowercase							  = ''''''
   __lowercase							  = False
   for token in tokens:
    # make sure that special tokens are not decoded using sentencepiece model
    if token in self.all_special_tokens:
     if not prev_is_special:
      out_string += " "
     out_string += self.sp_model.decode(lowercase__			) + token
     __lowercase							  = True
     __lowercase							  = []
    else:
     current_sub_tokens.append(lowercase__			)
     __lowercase							  = False
   out_string += self.sp_model.decode(lowercase__			)
   return out_string.strip()






  def 					SCREAMING_SNAKE_CASE      (      self   :     Dict					,lowercase__   :     str					,lowercase__   :     Optional[str] = None			):
   if not os.path.isdir(lowercase__			):
    logger.error(F"Vocabulary path ({save_directory}) should be a directory"			)
    return
   __lowercase							  = os.path.join(
       lowercase__					,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)

   if os.path.abspath(self.vocab_file			) != os.path.abspath(lowercase__			) and os.path.isfile(self.vocab_file			):
    copyfile(self.vocab_file					,lowercase__			)
   elif not os.path.isfile(self.vocab_file			):
    with open(lowercase__					,'''wb'''			) as fi:
     __lowercase							  = self.sp_model.serialized_model_proto()
     fi.write(lowercase__			)

   return (out_vocab_file,)




'''simple docstring'''






# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from packaging.version import Version, parse

from accelerate.commands.config.config_args import default_config_file, load_config_from_file


lowerCAmelCase__											=	'''Run commands across TPU VMs for initial setup before running `accelerate launch`.'''


def 		_A     (    A__=None      ):






 """simple docstring"""



 if subparsers is not None:
  __lowercase							  = subparsers.add_parser('''tpu-config'''     ,				description=_description      )
 else:
  __lowercase							  = argparse.ArgumentParser('''Accelerate tpu-config command'''     ,				description=_description      )
 # Core arguments
 __lowercase							  = parser.add_argument_group(
     '''Config Arguments'''     ,				'''Arguments that can be configured through `accelerate config`.'''      )
 config_args.add_argument(
     '''--config_file'''     ,				type=A__     ,				default=A__     ,				help='''Path to the config file to use for accelerate.'''     ,				)
 config_args.add_argument(
     '''--tpu_name'''     ,				default=A__     ,				help='''The name of the TPU to use. If not specified, will use the TPU specified in the config file.'''     ,				)
 config_args.add_argument(
     '''--tpu_zone'''     ,				default=A__     ,				help='''The zone of the TPU to use. If not specified, will use the zone specified in the config file.'''     ,				)
 __lowercase							  = parser.add_argument_group('''TPU Arguments'''     ,				'''Arguments for options ran inside the TPU.'''      )
 pod_args.add_argument(
     '''--use_alpha'''     ,				action='''store_true'''     ,				help='''Whether to use `gcloud alpha` when running the TPU training script instead of `gcloud`.'''     ,				)
 pod_args.add_argument(
     '''--command_file'''     ,				default=A__     ,				help='''The path to the file containing the commands to run on the pod on startup.'''     ,				)
 pod_args.add_argument(
     '''--command'''     ,				action='''append'''     ,				nargs='''+'''     ,				help='''A command to run on the pod. Can be passed multiple times.'''     ,				)
 pod_args.add_argument(
     '''--install_accelerate'''     ,				action='''store_true'''     ,				help='''Whether to install accelerate on the pod. Defaults to False.'''     ,				)
 pod_args.add_argument(
     '''--accelerate_version'''     ,				default='''latest'''     ,				help='''The version of accelerate to install on the pod. If not specified, will use the latest pypi version. Specify \'dev\' to install from GitHub.'''     ,				)
 pod_args.add_argument(
     '''--debug'''     ,				action='''store_true'''     ,				help='''If set, will print the command that would be run instead of running it.'''      )

 if subparsers is not None:
  parser.set_defaults(func=A__      )
 return parser


def 		_A     (    A__      ):






 """simple docstring"""



 __lowercase							  = None

 # Get the default from the config file if it exists.
 if args.config_file is not None or os.path.isfile(A__      ):
  __lowercase							  = load_config_from_file(args.config_file      )
  if not args.command_file and defaults.command_file is not None and not args.command:
   __lowercase							  = defaults.command_file
  if not args.command and defaults.commands is not None:
   __lowercase							  = defaults.commands
  if not args.tpu_name:
   __lowercase							  = defaults.tpu_name
  if not args.tpu_zone:
   __lowercase							  = defaults.tpu_zone
 if args.accelerate_version == "dev":
  __lowercase							  = '''git+https://github.com/huggingface/accelerate.git'''
 elif args.accelerate_version == "latest":
  __lowercase							  = '''accelerate -U'''
 elif isinstance(parse(args.accelerate_version      )     ,				A__      ):
  __lowercase							  = F"accelerate=={args.accelerate_version}"

 if not args.command_file and not args.command:
  raise ValueError('''You must specify either a command file or a command to run on the pod.'''      )

 if args.command_file:
  with open(args.command_file     ,				'''r'''      ) as f:
   __lowercase							  = [f.read().splitlines()]

    # To turn list of lists into list of strings
 if isinstance(args.command[0]     ,				A__      ):
  __lowercase							  = [line for cmd in args.command for line in cmd]
 # Default to the shared folder and install accelerate
 __lowercase							  = ['''cd /usr/share''']
 if args.install_accelerate:
  new_cmd += [F"pip install {args.accelerate_version}"]
 new_cmd += args.command
 __lowercase							  = '''; '''.join(A__      )

 # Then send it to gcloud
 # Eventually try to use google-api-core to do this instead of subprocess
 __lowercase							  = ['''gcloud''']
 if args.use_alpha:
  cmd += ["alpha"]
 cmd += [
     "compute",
     "tpus",
     "tpu-vm",
     "ssh",
     args.tpu_name,
     "--zone",
     args.tpu_zone,
     "--command",
     args.command,
     "--worker",
     "all",
 ]
 if args.debug:
  print(F"Running {' '.join(A__      )}"      )
  return
 subprocess.run(A__      )
 print('''Successfully setup pod.'''      )






def 		_A     (    ):






 """simple docstring"""



 __lowercase							  = tpu_command_parser()
 __lowercase							  = parser.parse_args()

 tpu_command_launcher(A__      )

