






import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


__a   :							Dict						= logging.get_logger(__name__)


__a   :							Dict						= {"vocab_file": "vocab.json", "merges_file": "merges.txt"}

# See all LED models at https://huggingface.co/models?filter=LED
__a   :							List[str]						= {
    "vocab_file": {
        "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json",
    },
    "merges_file": {
        "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json",
    },
}

__a   :							Optional[Any]						= {
    "allenai/led-base-16384": 16384,
}



@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def       _SCREAMING_SNAKE_CASE    (			)    ->   Optional[Any]:





    """simple docstring"""

    __A       =						(
        list(range(ord("""!"""     )      ,					ord("""~"""     ) + 1     )     ) + list(range(ord("""¡"""     )      ,					ord("""¬"""     ) + 1     )     ) + list(range(ord("""®"""     )      ,					ord("""ÿ"""     ) + 1     )     )
    )
    __A       =						bs[:]
    __A       =						0
    for b in range(2**8     ):
        if b not in bs:
            bs.append(__lowercase     )
            cs.append(2**8 + n     )
            n += 1
    __A       =						[chr(__lowercase     ) for n in cs]
    return dict(zip(__lowercase      ,					__lowercase     )     )



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      Union[str, Any]     )    ->   Union[str, Any]:





    """simple docstring"""

    __A       =						set()
    __A       =						word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)     )
        __A       =						char
    return pairs






class    __lowercase   (						lowercase_	):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE     =					VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE     =					PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE     =					["input_ids", "attention_mask"]




     def __init__(					self							:      Optional[Any]			,						UpperCamelCase_							:      List[Any]			,						UpperCamelCase_							:      Optional[Any]			,						UpperCamelCase_							:      Optional[Any]="replace"			,						UpperCamelCase_							:      Tuple="<s>"			,						UpperCamelCase_							:      Union[str, Any]="</s>"			,						UpperCamelCase_							:      Optional[int]="</s>"			,						UpperCamelCase_							:      List[Any]="<s>"			,						UpperCamelCase_							:      List[str]="<unk>"			,						UpperCamelCase_							:      str="<pad>"			,						UpperCamelCase_							:      int="<mask>"			,						UpperCamelCase_							:      Union[str, Any]=False			,						**UpperCamelCase_							:      List[str]			,						):


         """simple docstring"""



         __A       =						AddedToken(UpperCamelCase_			,						lstrip=UpperCamelCase_			,						rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_			,						UpperCamelCase_					) else bos_token
         __A       =						AddedToken(UpperCamelCase_			,						lstrip=UpperCamelCase_			,						rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_			,						UpperCamelCase_					) else eos_token
         __A       =						AddedToken(UpperCamelCase_			,						lstrip=UpperCamelCase_			,						rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_			,						UpperCamelCase_					) else sep_token
         __A       =						AddedToken(UpperCamelCase_			,						lstrip=UpperCamelCase_			,						rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_			,						UpperCamelCase_					) else cls_token
         __A       =						AddedToken(UpperCamelCase_			,						lstrip=UpperCamelCase_			,						rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_			,						UpperCamelCase_					) else unk_token
         __A       =						AddedToken(UpperCamelCase_			,						lstrip=UpperCamelCase_			,						rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_			,						UpperCamelCase_					) else pad_token

         # Mask token behave like a normal word, i.e. include the space before it
         __A       =						AddedToken(UpperCamelCase_			,						lstrip=UpperCamelCase_			,						rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_			,						UpperCamelCase_					) else mask_token

         super().__init__(
             errors=UpperCamelCase_			,						bos_token=UpperCamelCase_			,						eos_token=UpperCamelCase_			,						unk_token=UpperCamelCase_			,						sep_token=UpperCamelCase_			,						cls_token=UpperCamelCase_			,						pad_token=UpperCamelCase_			,						mask_token=UpperCamelCase_			,						add_prefix_space=UpperCamelCase_			,						**UpperCamelCase_			,						)

         with open(UpperCamelCase_			,						encoding="""utf-8"""					) as vocab_handle:
             __A       =						json.load(UpperCamelCase_					)
         __A       =						{v: k for k, v in self.encoder.items()}
         __A       =						errors  # how to handle errors in decoding
         __A       =						bytes_to_unicode()
         __A       =						{v: k for k, v in self.byte_encoder.items()}
         with open(UpperCamelCase_			,						encoding="""utf-8"""					) as merges_handle:
             __A       =						merges_handle.read().split("""\n"""					)[1:-1]
         __A       =						[tuple(merge.split()					) for merge in bpe_merges]
         __A       =						dict(zip(UpperCamelCase_			,						range(len(UpperCamelCase_					)					)					)					)
         __A       =						{}
         __A       =						add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         __A       =						re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""					)




     @property
     # Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
     def 		lowerCAmelCase_  (					self							:      Tuple					):


         """simple docstring"""



         return len(self.encoder					)




     def 		lowerCAmelCase_  (					self							:      Optional[Any]					):


         """simple docstring"""



         return dict(self.encoder			,						**self.added_tokens_encoder					)




     def 		lowerCAmelCase_  (					self							:      Dict			,						UpperCamelCase_							:      Any					):


         """simple docstring"""



         if token in self.cache:
             return self.cache[token]
         __A       =						tuple(UpperCamelCase_					)
         __A       =						get_pairs(UpperCamelCase_					)

         if not pairs:
             return token

         while True:
             __A       =						min(UpperCamelCase_			,						key=lambda UpperCamelCase_					: self.bpe_ranks.get(UpperCamelCase_			,						float("""inf"""					)					)					)
             if bigram not in self.bpe_ranks:
                 break
             __A     ,						__A       =						bigram
             __A       =						[]
             __A       =						0
             while i < len(UpperCamelCase_					):
                 try:
                     __A       =						word.index(UpperCamelCase_			,						UpperCamelCase_					)
                 except ValueError:
                     new_word.extend(word[i:]					)
                     break
                 else:
                     new_word.extend(word[i:j]					)
                     __A       =						j

                 if word[i] == first and i < len(UpperCamelCase_					) - 1 and word[i + 1] == second:
                     new_word.append(first + second					)
                     i += 2
                 else:
                     new_word.append(word[i]					)
                     i += 1
             __A       =						tuple(UpperCamelCase_					)
             __A       =						new_word
             if len(UpperCamelCase_					) == 1:
                 break
             else:
                 __A       =						get_pairs(UpperCamelCase_					)
         __A       =						""" """.join(UpperCamelCase_					)
         __A       =						word
         return word




     def 		lowerCAmelCase_  (					self							:      List[str]			,						UpperCamelCase_							:      str					):


         """simple docstring"""



         __A       =						[]
         for token in re.findall(self.pat			,						UpperCamelCase_					):
             __A       =						"""""".join(
                 self.byte_encoder[b] for b in token.encode("""utf-8"""					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
             bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase_					).split(""" """					)					)
         return bpe_tokens




     def 		lowerCAmelCase_  (					self							:      Any			,						UpperCamelCase_							:      Any					):


         """simple docstring"""



         return self.encoder.get(UpperCamelCase_			,						self.encoder.get(self.unk_token					)					)




     def 		lowerCAmelCase_  (					self							:      List[Any]			,						UpperCamelCase_							:      str					):


         """simple docstring"""



         return self.decoder.get(UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      Tuple			,						UpperCamelCase_							:      List[Any]					):


         """simple docstring"""



         __A       =						"""""".join(UpperCamelCase_					)
         __A       =						bytearray([self.byte_decoder[c] for c in text]					).decode("""utf-8"""			,						errors=self.errors					)
         return text




     def 		lowerCAmelCase_  (					self							:      str			,						UpperCamelCase_							:      str			,						UpperCamelCase_							:      Optional[str] = None					):


         """simple docstring"""



         if not os.path.isdir(UpperCamelCase_					):
             logger.error(F"Vocabulary path ({save_directory}) should be a directory"					)
             return
         __A       =						os.path.join(
             UpperCamelCase_			,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
         __A       =						os.path.join(
             UpperCamelCase_			,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]					)

         with open(UpperCamelCase_			,						"""w"""			,						encoding="""utf-8"""					) as f:
             f.write(json.dumps(self.encoder			,						indent=2			,						sort_keys=UpperCamelCase_			,						ensure_ascii=UpperCamelCase_					) + """\n"""					)

         __A       =						0
         with open(UpperCamelCase_			,						"""w"""			,						encoding="""utf-8"""					) as writer:
             writer.write("""#version: 0.2\n"""					)
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,						key=lambda UpperCamelCase_					: kv[1]					):
                 if index != token_index:
                     logger.warning(
                         F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                         """ Please check that the tokenizer is not corrupted!"""					)
                     __A       =						token_index
                 writer.write(""" """.join(UpperCamelCase_					) + """\n"""					)
                 index += 1

         return vocab_file, merge_file




     def 		lowerCAmelCase_  (					self							:      Dict			,						UpperCamelCase_							:      List[int]			,						UpperCamelCase_							:      Optional[List[int]] = None					):


         """simple docstring"""



         if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         __A       =						[self.cls_token_id]
         __A       =						[self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep




     def 		lowerCAmelCase_  (					self							:      List[Any]			,						UpperCamelCase_							:      List[int]			,						UpperCamelCase_							:      Optional[List[int]] = None			,						UpperCamelCase_							:      bool = False					):


         """simple docstring"""



         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=UpperCamelCase_			,						token_ids_a=UpperCamelCase_			,						already_has_special_tokens=UpperCamelCase_					)

         if token_ids_a is None:
             return [1] + ([0] * len(UpperCamelCase_					)) + [1]
         return [1] + ([0] * len(UpperCamelCase_					)) + [1, 1] + ([0] * len(UpperCamelCase_					)) + [1]




     def 		lowerCAmelCase_  (					self							:      Optional[int]			,						UpperCamelCase_							:      List[int]			,						UpperCamelCase_							:      Optional[List[int]] = None					):


         """simple docstring"""



         __A       =						[self.sep_token_id]
         __A       =						[self.cls_token_id]

         if token_ids_a is None:
             return len(cls + token_ids_a + sep					) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]




     def 		lowerCAmelCase_  (					self							:      List[Any]			,						UpperCamelCase_							:      Union[str, Any]			,						UpperCamelCase_							:      Union[str, Any]=False			,						**UpperCamelCase_							:      Optional[int]					):


         """simple docstring"""



         __A       =						kwargs.pop("""add_prefix_space"""			,						self.add_prefix_space					)
         if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase_					) > 0 and not text[0].isspace()):
             __A       =						""" """ + text
         return (text, kwargs)





     def 		lowerCAmelCase_  (					self							:      Any			,						UpperCamelCase_							:      Union[Dict[str, EncodedInput], BatchEncoding]			,						UpperCamelCase_							:      Optional[int] = None			,						UpperCamelCase_							:      PaddingStrategy = PaddingStrategy.DO_NOT_PAD			,						UpperCamelCase_							:      Optional[int] = None			,						UpperCamelCase_							:      Optional[bool] = None			,						):


         """simple docstring"""



         __A       =						super()._pad(
             encoded_inputs=UpperCamelCase_			,						max_length=UpperCamelCase_			,						padding_strategy=UpperCamelCase_			,						pad_to_multiple_of=UpperCamelCase_			,						return_attention_mask=UpperCamelCase_			,						)

         # Load from model defaults
         if return_attention_mask is None:
             __A       =						"""attention_mask""" in self.model_input_names

         if return_attention_mask and "global_attention_mask" in encoded_inputs:
             __A       =						encoded_inputs[self.model_input_names[0]]
             # `global_attention_mask` need to have the same length as other (sequential) inputs.
             __A       =						len(encoded_inputs["""global_attention_mask"""]					) != len(UpperCamelCase_					)

             if needs_to_be_padded:
                 __A       =						len(UpperCamelCase_					) - len(encoded_inputs["""global_attention_mask"""]					)

                 if self.padding_side == "right":
                     # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
                     __A       =						(
                         encoded_inputs["""global_attention_mask"""] + [-1] * difference
                     )
                 elif self.padding_side == "left":
                     __A       =						[-1] * difference + encoded_inputs[
                         """global_attention_mask"""
                     ]
                 else:
                     raise ValueError("""Invalid padding strategy:""" + str(self.padding_side					)					)

         return encoded_inputs












from ....configuration_utils import PretrainedConfig
from ....utils import logging


__a   :							List[Any]						= logging.get_logger(__name__)

# TODO: upload to AWS
__a   :							Union[str, Any]						= {
    "yjernite/retribert-base-uncased": (
        "https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/config.json"
    ),
}


class    __lowercase   (						lowercase_	):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE     =					"retribert"




     def __init__(					self							:      Any			,						UpperCamelCase_							:      int=30_522			,						UpperCamelCase_							:      Optional[int]=768			,						UpperCamelCase_							:      Union[str, Any]=8			,						UpperCamelCase_							:      Tuple=12			,						UpperCamelCase_							:      List[Any]=3_072			,						UpperCamelCase_							:      Tuple="gelu"			,						UpperCamelCase_							:      str=0.1			,						UpperCamelCase_							:      Dict=0.1			,						UpperCamelCase_							:      List[str]=512			,						UpperCamelCase_							:      Optional[int]=2			,						UpperCamelCase_							:      Tuple=0.02			,						UpperCamelCase_							:      List[str]=1e-12			,						UpperCamelCase_							:      Any=True			,						UpperCamelCase_							:      List[Any]=128			,						UpperCamelCase_							:      Union[str, Any]=0			,						**UpperCamelCase_							:      List[str]			,						):


         """simple docstring"""



         super().__init__(pad_token_id=UpperCamelCase_			,						**UpperCamelCase_					)

         __A       =						vocab_size
         __A       =						hidden_size
         __A       =						num_hidden_layers
         __A       =						num_attention_heads
         __A       =						hidden_act
         __A       =						intermediate_size
         __A       =						hidden_dropout_prob
         __A       =						attention_probs_dropout_prob
         __A       =						max_position_embeddings
         __A       =						type_vocab_size
         __A       =						initializer_range
         __A       =						layer_norm_eps
         __A       =						share_encoders
         __A       =						projection_dim




