'''simple docstring'''




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE       :     str     =  logging.get_logger(__name__)


__SCREAMING_SNAKE_CASE       :     Optional[int]     =  {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

__SCREAMING_SNAKE_CASE       :     Dict     =  {
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
}

__SCREAMING_SNAKE_CASE       :     Tuple     =  {
    """allenai/longformer-base-4096""": 4_096,
    """allenai/longformer-large-4096""": 4_096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4_096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4_096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4_096,
}

@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 	UpperCamelCase_							(					)    ->    Tuple:


       """simple docstring"""







       _UpperCAmelCase     :  Tuple							      =    (
           list(range(ord("!"    )			,	ord("~"    ) + 1    )    ) + list(range(ord("¡"    )			,	ord("¬"    ) + 1    )    ) + list(range(ord("®"    )			,	ord("ÿ"    ) + 1    )    )
       )
       _UpperCAmelCase     :  Tuple							      =    bs[:]
       _UpperCAmelCase     :  str							      =    0
       for b in range(2**8    ):
              if b not in bs:
                     bs.append(_UpperCAmelCase    )
                     cs.append(2**8 + n    )
                     n += 1
       _UpperCAmelCase     :  Optional[int]							      =    [chr(_UpperCAmelCase    ) for n in cs]
       return dict(zip(_UpperCAmelCase			,	_UpperCAmelCase    )    )

def 	UpperCamelCase_							(					_UpperCAmelCase  :      Any    )    ->    List[Any]:


       """simple docstring"""







       _UpperCAmelCase     :  Union[str, Any]							      =    set()
       _UpperCAmelCase     :  int							      =    word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)    )
              _UpperCAmelCase     :  Dict							      =    char
       return pairs


class 	lowerCamelCase_	(snake_case__    ):
      '''simple docstring'''

      __UpperCamelCase:  Any						=						VOCAB_FILES_NAMES
      __UpperCamelCase:  str						=						PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase:  Optional[Any]						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase:  Tuple						=						["input_ids", "attention_mask"]




      def __init__(       self						:				int						,					A						:				List[Any]						,					A						:				str						,					A						:				Union[str, Any]="replace"						,					A						:				Any="<s>"						,					A						:				Tuple="</s>"						,					A						:				Tuple="</s>"						,					A						:				Dict="<s>"						,					A						:				str="<unk>"						,					A						:				Dict="<pad>"						,					A						:				List[Any]="<mask>"						,					A						:				Union[str, Any]=False						,					**A						:				List[str]						,					):
             _UpperCAmelCase     :  str							      =    AddedToken(A						,					lstrip=A						,					rstrip=A       ) if isinstance(A						,					A       ) else bos_token
             _UpperCAmelCase     :  List[str]							      =    AddedToken(A						,					lstrip=A						,					rstrip=A       ) if isinstance(A						,					A       ) else eos_token
             _UpperCAmelCase     :  Dict							      =    AddedToken(A						,					lstrip=A						,					rstrip=A       ) if isinstance(A						,					A       ) else sep_token
             _UpperCAmelCase     :  Union[str, Any]							      =    AddedToken(A						,					lstrip=A						,					rstrip=A       ) if isinstance(A						,					A       ) else cls_token
             _UpperCAmelCase     :  List[Any]							      =    AddedToken(A						,					lstrip=A						,					rstrip=A       ) if isinstance(A						,					A       ) else unk_token
             _UpperCAmelCase     :  Optional[Any]							      =    AddedToken(A						,					lstrip=A						,					rstrip=A       ) if isinstance(A						,					A       ) else pad_token

             # Mask token behave like a normal word, i.e. include the space before it
             _UpperCAmelCase     :  str							      =    AddedToken(A						,					lstrip=A						,					rstrip=A       ) if isinstance(A						,					A       ) else mask_token

             super().__init__(
                 errors=A						,					bos_token=A						,					eos_token=A						,					unk_token=A						,					sep_token=A						,					cls_token=A						,					pad_token=A						,					mask_token=A						,					add_prefix_space=A						,					**A						,					)

             with open(A						,					encoding="utf-8"       ) as vocab_handle:
                    _UpperCAmelCase     :  Union[str, Any]							      =    json.load(A       )
             _UpperCAmelCase     :  Any							      =    {v: k for k, v in self.encoder.items()}
             _UpperCAmelCase     :  Tuple							      =    errors  # how to handle errors in decoding
             _UpperCAmelCase     :  Any							      =    bytes_to_unicode()
             _UpperCAmelCase     :  Dict							      =    {v: k for k, v in self.byte_encoder.items()}
             with open(A						,					encoding="utf-8"       ) as merges_handle:
                    _UpperCAmelCase     :  Any							      =    merges_handle.read().split("\n"       )[1:-1]
             _UpperCAmelCase     :  List[Any]							      =    [tuple(merge.split()       ) for merge in bpe_merges]
             _UpperCAmelCase     :  Any							      =    dict(zip(A						,					range(len(A       )       )       )       )
             _UpperCAmelCase     :  Any							      =    {}
             _UpperCAmelCase     :  List[Any]							      =    add_prefix_space

             # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
             _UpperCAmelCase     :  Optional[Any]							      =    re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"       )




      @property
      def      _A	(       self						:				Tuple       ):
             return len(self.encoder       )




      def      _A	(       self						:				Optional[Any]       ):
             return dict(self.encoder						,					**self.added_tokens_encoder       )




      def      _A	(       self						:				Any						,					A						:				Optional[int]       ):
             if token in self.cache:
                    return self.cache[token]
             _UpperCAmelCase     :  int							      =    tuple(A       )
             _UpperCAmelCase     :  List[str]							      =    get_pairs(A       )

             if not pairs:
                    return token

             while True:
                    _UpperCAmelCase     :  str							      =    min(A						,					key=lambda A       : self.bpe_ranks.get(A						,					float("inf"       )       )       )
                    if bigram not in self.bpe_ranks:
                           break
                    _UpperCAmelCase      ,  _UpperCAmelCase     :  Tuple							      =    bigram
                    _UpperCAmelCase     :  List[Any]							      =    []
                    _UpperCAmelCase     :  Dict							      =    0
                    while i < len(A       ):
                           try:
                                  _UpperCAmelCase     :  str							      =    word.index(A						,					A       )
                           except ValueError:
                                  new_word.extend(word[i:]       )
                                  break
                           else:
                                  new_word.extend(word[i:j]       )
                                  _UpperCAmelCase     :  Union[str, Any]							      =    j

                           if word[i] == first and i < len(A       ) - 1 and word[i + 1] == second:
                                  new_word.append(first + second       )
                                  i += 2
                           else:
                                  new_word.append(word[i]       )
                                  i += 1
                    _UpperCAmelCase     :  Optional[Any]							      =    tuple(A       )
                    _UpperCAmelCase     :  List[Any]							      =    new_word
                    if len(A       ) == 1:
                           break
                    else:
                           _UpperCAmelCase     :  Any							      =    get_pairs(A       )
             _UpperCAmelCase     :  Union[str, Any]							      =    " ".join(A       )
             _UpperCAmelCase     :  str							      =    word
             return word




      def      _A	(       self						:				str						,					A						:				Dict       ):
             _UpperCAmelCase     :  Tuple							      =    []
             for token in re.findall(self.pat						,					A       ):
                    _UpperCAmelCase     :  Any							      =    "".join(
                        self.byte_encoder[b] for b in token.encode("utf-8"       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                    bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A       ).split(" "       )       )
             return bpe_tokens




      def      _A	(       self						:				Any						,					A						:				Dict       ):
             return self.encoder.get(A						,					self.encoder.get(self.unk_token       )       )




      def      _A	(       self						:				Tuple						,					A						:				Optional[Any]       ):
             return self.decoder.get(A       )




      def      _A	(       self						:				Tuple						,					A						:				str       ):
             _UpperCAmelCase     :  Optional[Any]							      =    "".join(A       )
             _UpperCAmelCase     :  str							      =    bytearray([self.byte_decoder[c] for c in text]       ).decode("utf-8"						,					errors=self.errors       )
             return text




      def      _A	(       self						:				int						,					A						:				str						,					A						:				Optional[str] = None       ):
             if not os.path.isdir(A       ):
                    logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
                    return
             _UpperCAmelCase     :  str							      =    os.path.join(
                 A						,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )
             _UpperCAmelCase     :  int							      =    os.path.join(
                 A						,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]       )

             with open(A						,					"w"						,					encoding="utf-8"       ) as f:
                    f.write(json.dumps(self.encoder						,					indent=2						,					sort_keys=A						,					ensure_ascii=A       ) + "\n"       )

             _UpperCAmelCase     :  Optional[Any]							      =    0
             with open(A						,					"w"						,					encoding="utf-8"       ) as writer:
                    writer.write("#version: 0.2\n"       )
                    for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,					key=lambda A       : kv[1]       ):
                           if index != token_index:
                                  logger.warning(
                                      F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                                      " Please check that the tokenizer is not corrupted!"       )
                                  _UpperCAmelCase     :  Any							      =    token_index
                           writer.write(" ".join(A       ) + "\n"       )
                           index += 1

             return vocab_file, merge_file




      def      _A	(       self						:				List[Any]						,					A						:				List[int]						,					A						:				Optional[List[int]] = None       ):
             if token_ids_a is None:
                    return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
             _UpperCAmelCase     :  int							      =    [self.cls_token_id]
             _UpperCAmelCase     :  Tuple							      =    [self.sep_token_id]
             return cls + token_ids_a + sep + sep + token_ids_a + sep




      def      _A	(       self						:				Optional[int]						,					A						:				List[int]						,					A						:				Optional[List[int]] = None						,					A						:				bool = False       ):
             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=A						,					token_ids_a=A						,					already_has_special_tokens=A       )

             if token_ids_a is None:
                    return [1] + ([0] * len(A       )) + [1]
             return [1] + ([0] * len(A       )) + [1, 1] + ([0] * len(A       )) + [1]




      def      _A	(       self						:				Union[str, Any]						,					A						:				List[int]						,					A						:				Optional[List[int]] = None       ):
             _UpperCAmelCase     :  int							      =    [self.sep_token_id]
             _UpperCAmelCase     :  int							      =    [self.cls_token_id]

             if token_ids_a is None:
                    return len(cls + token_ids_a + sep       ) * [0]
             return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]





      def      _A	(       self						:				List[str]						,					A						:				Optional[int]						,					A						:				Optional[Any]=False						,					**A						:				List[str]       ):
             _UpperCAmelCase     :  Optional[int]							      =    kwargs.pop("add_prefix_space"						,					self.add_prefix_space       )
             if (is_split_into_words or add_prefix_space) and (len(A       ) > 0 and not text[0].isspace()):
                    _UpperCAmelCase     :  Union[str, Any]							      =    " " + text
             return (text, kwargs)








from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase_    =				logging.get_logger(__name__)

lowerCAmelCase_    =				{
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}



class 					__lowerCAmelCase       (			_a     ):
			lowerCamelCase_     :     Any       			=      '''megatron-bert'''






			def __init__(self					,							__magic_name__=2_9056					,							__magic_name__=1024					,							__magic_name__=24					,							__magic_name__=16					,							__magic_name__=4096					,							__magic_name__="gelu"					,							__magic_name__=0.1					,							__magic_name__=0.1					,							__magic_name__=512					,							__magic_name__=2					,							__magic_name__=0.02					,							__magic_name__=1e-12					,							__magic_name__=0					,							__magic_name__="absolute"					,							__magic_name__=True					,							**__magic_name__					,							)     ->   Any:


							'''simple docstring'''



							super().__init__(pad_token_id=__magic_name__					,							**__magic_name__    )

							snake_case_						:    Union[str, Any]				=  vocab_size
							snake_case_						:    Dict				=  hidden_size
							snake_case_						:    Dict				=  num_hidden_layers
							snake_case_						:    Optional[int]				=  num_attention_heads
							snake_case_						:    int				=  hidden_act
							snake_case_						:    List[str]				=  intermediate_size
							snake_case_						:    Dict				=  hidden_dropout_prob
							snake_case_						:    str				=  attention_probs_dropout_prob
							snake_case_						:    str				=  max_position_embeddings
							snake_case_						:    Any				=  type_vocab_size
							snake_case_						:    int				=  initializer_range
							snake_case_						:    int				=  layer_norm_eps
							snake_case_						:    List[str]				=  position_embedding_type
							snake_case_						:    Dict				=  use_cache
