





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case__					:							int													=     logging.get_logger(__name__)


snake_case__					:							str													=     {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

snake_case__					:							Any													=     {
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
}

snake_case__					:							Optional[Any]													=     {
    'allenai/longformer-base-4096': 4096,
    'allenai/longformer-large-4096': 4096,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4096,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4096,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4096,
}



@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def _a				(	)	->						Tuple:







      '''simple docstring'''






      __A          =				(
          list(range(ord('''!''' )					,				ord('''~''' ) + 1 ) ) + list(range(ord('''¡''' )					,				ord('''¬''' ) + 1 ) ) + list(range(ord('''®''' )					,				ord('''ÿ''' ) + 1 ) )
      )
      __A          =				bs[:]
      __A          =				0
      for b in range(2**8 ):
            if b not in bs:
                  bs.append(lowerCamelCase )
                  cs.append(2**8 + n )
                  n += 1
      __A          =				[chr(lowerCamelCase ) for n in cs]
      return dict(zip(lowerCamelCase					,				lowerCamelCase ) )



def _a				(	lowerCamelCase:   Dict )	->						Any:







      '''simple docstring'''






      __A          =				set()
      __A          =				word[0]
      for char in word[1:]:
            pairs.add((prev_char, char) )
            __A          =				char
      return pairs







class A_					(			_lowerCamelCase   ):
   lowerCAmelCase__        = VOCAB_FILES_NAMES
   lowerCAmelCase__        = PRETRAINED_VOCAB_FILES_MAP
   lowerCAmelCase__        = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCAmelCase__        = ["""input_ids""", """attention_mask"""]




   def __init__(self	:Optional[int]    ,   _UpperCamelCase	:Optional[int]    ,   _UpperCamelCase	:Tuple    ,   _UpperCamelCase	:Tuple="replace"    ,   _UpperCamelCase	:Optional[Any]="<s>"    ,   _UpperCamelCase	:Any="</s>"    ,   _UpperCamelCase	:Dict="</s>"    ,   _UpperCamelCase	:Optional[int]="<s>"    ,   _UpperCamelCase	:List[Any]="<unk>"    ,   _UpperCamelCase	:Tuple="<pad>"    ,   _UpperCamelCase	:Union[str, Any]="<mask>"    ,   _UpperCamelCase	:Any=False    ,   **_UpperCamelCase	:List[str]    ,   )->					Union[str, Any]:
         __A          =				AddedToken(_UpperCamelCase    ,   lstrip=_UpperCamelCase    ,   rstrip=_UpperCamelCase ) if isinstance(_UpperCamelCase    ,   _UpperCamelCase ) else bos_token
         __A          =				AddedToken(_UpperCamelCase    ,   lstrip=_UpperCamelCase    ,   rstrip=_UpperCamelCase ) if isinstance(_UpperCamelCase    ,   _UpperCamelCase ) else eos_token
         __A          =				AddedToken(_UpperCamelCase    ,   lstrip=_UpperCamelCase    ,   rstrip=_UpperCamelCase ) if isinstance(_UpperCamelCase    ,   _UpperCamelCase ) else sep_token
         __A          =				AddedToken(_UpperCamelCase    ,   lstrip=_UpperCamelCase    ,   rstrip=_UpperCamelCase ) if isinstance(_UpperCamelCase    ,   _UpperCamelCase ) else cls_token
         __A          =				AddedToken(_UpperCamelCase    ,   lstrip=_UpperCamelCase    ,   rstrip=_UpperCamelCase ) if isinstance(_UpperCamelCase    ,   _UpperCamelCase ) else unk_token
         __A          =				AddedToken(_UpperCamelCase    ,   lstrip=_UpperCamelCase    ,   rstrip=_UpperCamelCase ) if isinstance(_UpperCamelCase    ,   _UpperCamelCase ) else pad_token

         # Mask token behave like a normal word, i.e. include the space before it
         __A          =				AddedToken(_UpperCamelCase    ,   lstrip=_UpperCamelCase    ,   rstrip=_UpperCamelCase ) if isinstance(_UpperCamelCase    ,   _UpperCamelCase ) else mask_token

         super().__init__(
             errors=_UpperCamelCase    ,   bos_token=_UpperCamelCase    ,   eos_token=_UpperCamelCase    ,   unk_token=_UpperCamelCase    ,   sep_token=_UpperCamelCase    ,   cls_token=_UpperCamelCase    ,   pad_token=_UpperCamelCase    ,   mask_token=_UpperCamelCase    ,   add_prefix_space=_UpperCamelCase    ,   **_UpperCamelCase    ,   )

         with open(_UpperCamelCase    ,   encoding='''utf-8''' ) as vocab_handle:
               __A          =				json.load(_UpperCamelCase )
         __A          =				{v: k for k, v in self.encoder.items()}
         __A          =				errors  # how to handle errors in decoding
         __A          =				bytes_to_unicode()
         __A          =				{v: k for k, v in self.byte_encoder.items()}
         with open(_UpperCamelCase    ,   encoding='''utf-8''' ) as merges_handle:
               __A          =				merges_handle.read().split('''\n''' )[1:-1]
         __A          =				[tuple(merge.split() ) for merge in bpe_merges]
         __A          =				dict(zip(_UpperCamelCase    ,   range(len(_UpperCamelCase ) ) ) )
         __A          =				{}
         __A          =				add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         __A          =				re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+''' )




   @property
   def _lowerCAmelCase      (self	:Tuple )->					Tuple:
         return len(self.encoder )




   def _lowerCAmelCase      (self	:Optional[int] )->					Optional[int]:
         return dict(self.encoder    ,   **self.added_tokens_encoder )




   def _lowerCAmelCase      (self	:Dict    ,   _UpperCamelCase	:Tuple )->					Optional[int]:
         if token in self.cache:
               return self.cache[token]
         __A          =				tuple(_UpperCamelCase )
         __A          =				get_pairs(_UpperCamelCase )

         if not pairs:
               return token

         while True:
               __A          =				min(_UpperCamelCase    ,   key=lambda _UpperCamelCase : self.bpe_ranks.get(_UpperCamelCase    ,   float('''inf''' ) ) )
               if bigram not in self.bpe_ranks:
                     break
               __A   ,						__A          =				bigram
               __A          =				[]
               __A          =				0
               while i < len(_UpperCamelCase ):
                     try:
                           __A          =				word.index(_UpperCamelCase    ,   _UpperCamelCase )
                     except ValueError:
                           new_word.extend(word[i:] )
                           break
                     else:
                           new_word.extend(word[i:j] )
                           __A          =				j

                     if word[i] == first and i < len(_UpperCamelCase ) - 1 and word[i + 1] == second:
                           new_word.append(first + second )
                           i += 2
                     else:
                           new_word.append(word[i] )
                           i += 1
               __A          =				tuple(_UpperCamelCase )
               __A          =				new_word
               if len(_UpperCamelCase ) == 1:
                     break
               else:
                     __A          =				get_pairs(_UpperCamelCase )
         __A          =				''' '''.join(_UpperCamelCase )
         __A          =				word
         return word




   def _lowerCAmelCase      (self	:Dict    ,   _UpperCamelCase	:List[str] )->					Dict:
         __A          =				[]
         for token in re.findall(self.pat    ,   _UpperCamelCase ):
               __A          =				''''''.join(
                   self.byte_encoder[b] for b in token.encode('''utf-8''' ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_UpperCamelCase ).split(''' ''' ) )
         return bpe_tokens




   def _lowerCAmelCase      (self	:List[Any]    ,   _UpperCamelCase	:Dict )->					Any:
         return self.encoder.get(_UpperCamelCase    ,   self.encoder.get(self.unk_token ) )




   def _lowerCAmelCase      (self	:Any    ,   _UpperCamelCase	:Optional[int] )->					str:
         return self.decoder.get(_UpperCamelCase )




   def _lowerCAmelCase      (self	:Tuple    ,   _UpperCamelCase	:Tuple )->					Any:
         __A          =				''''''.join(_UpperCamelCase )
         __A          =				bytearray([self.byte_decoder[c] for c in text] ).decode('''utf-8'''    ,   errors=self.errors )
         return text




   def _lowerCAmelCase      (self	:Optional[int]    ,   _UpperCamelCase	:str    ,   _UpperCamelCase	:Optional[str] = None )->					Tuple[str]:
         if not os.path.isdir(_UpperCamelCase ):
               logger.error(f"""Vocabulary path ({save_directory}) should be a directory""" )
               return
         __A          =				os.path.join(
             _UpperCamelCase    ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )
         __A          =				os.path.join(
             _UpperCamelCase    ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''] )

         with open(_UpperCamelCase    ,   '''w'''    ,   encoding='''utf-8''' ) as f:
               f.write(json.dumps(self.encoder    ,   indent=2    ,   sort_keys=_UpperCamelCase    ,   ensure_ascii=_UpperCamelCase ) + '''\n''' )

         __A          =				0
         with open(_UpperCamelCase    ,   '''w'''    ,   encoding='''utf-8''' ) as writer:
               writer.write('''#version: 0.2\n''' )
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,   key=lambda _UpperCamelCase : kv[1] ):
                     if index != token_index:
                           logger.warning(
                               f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                               ''' Please check that the tokenizer is not corrupted!''' )
                           __A          =				token_index
                     writer.write(''' '''.join(_UpperCamelCase ) + '''\n''' )
                     index += 1

         return vocab_file, merge_file




   def _lowerCAmelCase      (self	:Optional[int]    ,   _UpperCamelCase	:List[int]    ,   _UpperCamelCase	:Optional[List[int]] = None )->					List[int]:
         if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         __A          =				[self.cls_token_id]
         __A          =				[self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep




   def _lowerCAmelCase      (self	:Optional[int]    ,   _UpperCamelCase	:List[int]    ,   _UpperCamelCase	:Optional[List[int]] = None    ,   _UpperCamelCase	:bool = False )->					List[int]:
         if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=_UpperCamelCase    ,   token_ids_a=_UpperCamelCase    ,   already_has_special_tokens=_UpperCamelCase )

         if token_ids_a is None:
               return [1] + ([0] * len(_UpperCamelCase )) + [1]
         return [1] + ([0] * len(_UpperCamelCase )) + [1, 1] + ([0] * len(_UpperCamelCase )) + [1]




   def _lowerCAmelCase      (self	:List[Any]    ,   _UpperCamelCase	:List[int]    ,   _UpperCamelCase	:Optional[List[int]] = None )->					List[int]:
         __A          =				[self.sep_token_id]
         __A          =				[self.cls_token_id]

         if token_ids_a is None:
               return len(cls + token_ids_a + sep ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]






   def _lowerCAmelCase      (self	:List[str]    ,   _UpperCamelCase	:str    ,   _UpperCamelCase	:int=False    ,   **_UpperCamelCase	:Union[str, Any] )->					Optional[int]:
         __A          =				kwargs.pop('''add_prefix_space'''    ,   self.add_prefix_space )
         if (is_split_into_words or add_prefix_space) and (len(_UpperCamelCase ) > 0 and not text[0].isspace()):
               __A          =				''' ''' + text
         return (text, kwargs)












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


snake_case__					:							Optional[Any]													=     {'configuration_ibert': ['IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'IBertConfig', 'IBertOnnxConfig']}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       snake_case__					:							List[str]													=     [
           'IBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
           'IBertForMaskedLM',
           'IBertForMultipleChoice',
           'IBertForQuestionAnswering',
           'IBertForSequenceClassification',
           'IBertForTokenClassification',
           'IBertModel',
           'IBertPreTrainedModel',
       ]

if TYPE_CHECKING:
       from .configuration_ibert import IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, IBertConfig, IBertOnnxConfig

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_ibert import (
                  IBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  IBertForMaskedLM,
                  IBertForMultipleChoice,
                  IBertForQuestionAnswering,
                  IBertForSequenceClassification,
                  IBertForTokenClassification,
                  IBertModel,
                  IBertPreTrainedModel,
              )

else:
       import sys

       snake_case__					:							Dict													=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





