

'''simple docstring'''



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_big_bird import BigBirdTokenizer
else:
  UpperCAmelCase :       Optional[int]			   =     None

UpperCAmelCase :       Any			   =     logging.get_logger(__name__)
UpperCAmelCase :       Dict			   =     {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase :       Optional[int]			   =     {
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    },
    'tokenizer_file': {
        'google/bigbird-roberta-base': (
            'https://huggingface.co/google/bigbird-roberta-base/resolve/main/tokenizer.json'
        ),
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/tokenizer.json'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase :       Dict			   =     {
    'google/bigbird-roberta-base': 4_0_9_6,
    'google/bigbird-roberta-large': 4_0_9_6,
    'google/bigbird-base-trivia-itc': 4_0_9_6,
}


UpperCAmelCase :       Union[str, Any]			   =     '▁'







class 						lowerCAmelCase__   (      a					):



 """simple docstring"""



 lowerCAmelCase__  					=      VOCAB_FILES_NAMES
 lowerCAmelCase__  					=      PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__  					=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__  					=      BigBirdTokenizer
 lowerCAmelCase__  					=      ["input_ids", "attention_mask"]
 lowerCAmelCase__  					=      []
 def __init__(		self					:       int					,							__SCREAMING_SNAKE_CASE					:       Dict=None					,							__SCREAMING_SNAKE_CASE					:       Any=None					,							__SCREAMING_SNAKE_CASE					:       Tuple="<unk>"					,							__SCREAMING_SNAKE_CASE					:       Tuple="<s>"					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]="</s>"					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]="<pad>"					,							__SCREAMING_SNAKE_CASE					:       List[str]="[SEP]"					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]="[MASK]"					,							__SCREAMING_SNAKE_CASE					:       List[Any]="[CLS]"					,							**__SCREAMING_SNAKE_CASE					:       Any					,							)  -> Dict:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      AddedToken(__SCREAMING_SNAKE_CASE					,							lstrip=__SCREAMING_SNAKE_CASE					,							rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			) else bos_token
      __SCREAMING_SNAKE_CASE     =      AddedToken(__SCREAMING_SNAKE_CASE					,							lstrip=__SCREAMING_SNAKE_CASE					,							rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			) else eos_token
      __SCREAMING_SNAKE_CASE     =      AddedToken(__SCREAMING_SNAKE_CASE					,							lstrip=__SCREAMING_SNAKE_CASE					,							rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			) else unk_token
      __SCREAMING_SNAKE_CASE     =      AddedToken(__SCREAMING_SNAKE_CASE					,							lstrip=__SCREAMING_SNAKE_CASE					,							rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			) else pad_token
      __SCREAMING_SNAKE_CASE     =      AddedToken(__SCREAMING_SNAKE_CASE					,							lstrip=__SCREAMING_SNAKE_CASE					,							rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			) else cls_token
      __SCREAMING_SNAKE_CASE     =      AddedToken(__SCREAMING_SNAKE_CASE					,							lstrip=__SCREAMING_SNAKE_CASE					,							rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			) else sep_token

      # Mask token behave like a normal word, i.e. include the space before it
      __SCREAMING_SNAKE_CASE     =      AddedToken(__SCREAMING_SNAKE_CASE					,							lstrip=__SCREAMING_SNAKE_CASE					,							rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			) else mask_token

      super().__init__(
          __SCREAMING_SNAKE_CASE					,							tokenizer_file=__SCREAMING_SNAKE_CASE					,							bos_token=__SCREAMING_SNAKE_CASE					,							eos_token=__SCREAMING_SNAKE_CASE					,							unk_token=__SCREAMING_SNAKE_CASE					,							sep_token=__SCREAMING_SNAKE_CASE					,							pad_token=__SCREAMING_SNAKE_CASE					,							cls_token=__SCREAMING_SNAKE_CASE					,							mask_token=__SCREAMING_SNAKE_CASE					,							**__SCREAMING_SNAKE_CASE					,							)

      __SCREAMING_SNAKE_CASE     =      vocab_file
      __SCREAMING_SNAKE_CASE     =      False if not self.vocab_file else True
 def       UpperCAmelCase__		(		self					:       Dict					,							__SCREAMING_SNAKE_CASE					:       List[int]					,							__SCREAMING_SNAKE_CASE					:       Optional[List[int]] = None			)  -> List[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      [self.sep_token_id]
      __SCREAMING_SNAKE_CASE     =      [self.cls_token_id]
      if token_ids_a is None:
           return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep
 def       UpperCAmelCase__		(		self					:       Tuple					,							__SCREAMING_SNAKE_CASE					:       List[int]					,							__SCREAMING_SNAKE_CASE					:       Optional[List[int]] = None					,							__SCREAMING_SNAKE_CASE					:       bool = False			)  -> List[int]:


      """simple docstring"""




      if already_has_special_tokens:
           if token_ids_a is not None:
                raise ValueError(
                    """You should not supply a second sequence if the provided sequence of """
                    """ids is already formatted with special tokens for the model."""			)
           return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

      if token_ids_a is None:
           return [1] + ([0] * len(__SCREAMING_SNAKE_CASE			)) + [1]
      return [1] + ([0] * len(__SCREAMING_SNAKE_CASE			)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE			)) + [1]
 def       UpperCAmelCase__		(		self					:       Tuple					,							__SCREAMING_SNAKE_CASE					:       List[int]					,							__SCREAMING_SNAKE_CASE					:       Optional[List[int]] = None			)  -> List[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      [self.sep_token_id]
      __SCREAMING_SNAKE_CASE     =      [self.cls_token_id]

      if token_ids_a is None:
           return len(cls + token_ids_a + sep			) * [0]
      return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]


 def       UpperCAmelCase__		(		self					:       List[Any]					,							__SCREAMING_SNAKE_CASE					:       str					,							__SCREAMING_SNAKE_CASE					:       Optional[str] = None			)  -> Tuple[str]:


      """simple docstring"""




      if not self.can_save_slow_tokenizer:
           raise ValueError(
               """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
               """tokenizer."""			)

      if not os.path.isdir(__SCREAMING_SNAKE_CASE			):
           logger.error(f'Vocabulary path ({save_directory}) should be a directory'			)
           return
      __SCREAMING_SNAKE_CASE     =      os.path.join(
          __SCREAMING_SNAKE_CASE					,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

      if os.path.abspath(self.vocab_file			) != os.path.abspath(__SCREAMING_SNAKE_CASE			):
           copyfile(self.vocab_file					,							__SCREAMING_SNAKE_CASE			)

      return (out_vocab_file,)









'''simple docstring'''



# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from ...utils.dataclasses import (
    ComputeEnvironment,
    DistributedType,
    DynamoBackend,
    PrecisionType,
    SageMakerDistributedType,
)
from ..menu import BulletMenu


UpperCAmelCase :       Any			   =     [
    'EAGER',
    'AOT_EAGER',
    'INDUCTOR',
    'NVFUSER',
    'AOT_NVFUSER',
    'AOT_CUDAGRAPHS',
    'OFI',
    'FX2TRT',
    'ONNXRT',
    'IPEX',
]







def    a__       (  a__      ,    a__=None      ,    a__=None      ,    a__=None						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      True
     while ask_again:
          __SCREAMING_SNAKE_CASE     =      input(a__						)
          try:
               if default is not None and len(a__						) == 0:
                    return default
               return convert_value(a__						) if convert_value is not None else result
          except Exception:
               if error_message is not None:
                    print(a__						)







def    a__       (  a__      ,    a__=[]      ,    a__=None      ,    a__=0						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      BulletMenu(a__      ,    a__						)
     __SCREAMING_SNAKE_CASE     =      menu.run(default_choice=a__						)
     return convert_value(a__						) if convert_value is not None else result







def    a__       (  a__						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      int(a__						)
     return ComputeEnvironment(["""LOCAL_MACHINE""", """AMAZON_SAGEMAKER"""][value]						)







def    a__       (  a__						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      int(a__						)
     return DistributedType(["""NO""", """MULTI_CPU""", """MULTI_XPU""", """MULTI_GPU""", """MULTI_NPU""", """TPU"""][value]						)







def    a__       (  a__						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      int(a__						)
     return DynamoBackend(DYNAMO_BACKENDS[value]						).value







def    a__       (  a__						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      int(a__						)
     return PrecisionType(["""no""", """fp16""", """bf16""", """fp8"""][value]						)







def    a__       (  a__						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      int(a__						)
     return SageMakerDistributedType(["""NO""", """DATA_PARALLEL""", """MODEL_PARALLEL"""][value]						)







def    a__       (  a__						):




     """simple docstring"""


     return {"yes": True, "no": False}[value.lower()]







class 						lowerCAmelCase__   (      argparse.RawDescriptionHelpFormatter					):



 """simple docstring"""



 def       UpperCAmelCase__		(		self					:       str					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]					,							__SCREAMING_SNAKE_CASE					:       Dict					,							__SCREAMING_SNAKE_CASE					:       Optional[int]					,							__SCREAMING_SNAKE_CASE					:       List[Any]			)  -> Dict:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      super()._format_usage(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      usage.replace("""<command> [<args>] """					,							""""""			)
      return usage






