


"""simple docstring"""






import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase    :     Optional[int]   						=		logging.get_logger(__name__)


UpperCAmelCase    :     Optional[int]   						=		{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

UpperCAmelCase    :     List[str]   						=		{
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
}

UpperCAmelCase    :     Dict   						=		{
    'allenai/longformer-base-4096': 4096,
    'allenai/longformer-large-4096': 4096,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4096,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4096,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4096,
}
@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def      lowerCamelCase       (     )		->       int:

							'''simple docstring'''



							__UpperCAmelCase   :						List[str]      							=					(
							    list(range(ord("""!"""							)       ,			ord("""~"""							) + 1							)							) + list(range(ord("""¡"""							)       ,			ord("""¬"""							) + 1							)							) + list(range(ord("""®"""							)       ,			ord("""ÿ"""							) + 1							)							)
							)
							__UpperCAmelCase   :						List[Any]      							=					bs[:]
							__UpperCAmelCase   :						Union[str, Any]      							=					0
							for b in range(2**8							):
														if b not in bs:
																					bs.append(_UpperCamelCase							)
																					cs.append(2**8 + n							)
																					n += 1
							__UpperCAmelCase   :						List[Any]      							=					[chr(_UpperCamelCase							) for n in cs]
							return dict(zip(_UpperCamelCase       ,			_UpperCamelCase							)							)
def      lowerCamelCase       (     _UpperCamelCase		:      str							)		->       Dict:

							'''simple docstring'''



							__UpperCAmelCase   :						Optional[int]      							=					set()
							__UpperCAmelCase   :						Union[str, Any]      							=					word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)							)
														__UpperCAmelCase   :						Union[str, Any]      							=					char
							return pairs





class      lowerCamelCase__   (     A				):







					"""simple docstring"""


					__a	       =							VOCAB_FILES_NAMES
					__a	       =							PRETRAINED_VOCAB_FILES_MAP
					__a	       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__a	       =							["""input_ids""", """attention_mask"""]






					def __init__(       self    : List[str] ,       UpperCamelCase    : List[Any] ,       UpperCamelCase    : Any ,       UpperCamelCase    : Optional[Any]="replace" ,       UpperCamelCase    : Union[str, Any]="<s>" ,       UpperCamelCase    : Dict="</s>" ,       UpperCamelCase    : Optional[Any]="</s>" ,       UpperCamelCase    : Optional[int]="<s>" ,       UpperCamelCase    : Optional[Any]="<unk>" ,       UpperCamelCase    : int="<pad>" ,       UpperCamelCase    : Union[str, Any]="<mask>" ,       UpperCamelCase    : Optional[Any]=False ,       **UpperCamelCase    : List[Any] ,       ):
												'''simple docstring'''






												__UpperCAmelCase   :						str      							=					AddedToken(UpperCamelCase ,       lstrip=UpperCamelCase ,       rstrip=UpperCamelCase  ) if isinstance(UpperCamelCase ,       UpperCamelCase  ) else bos_token
												__UpperCAmelCase   :						Tuple      							=					AddedToken(UpperCamelCase ,       lstrip=UpperCamelCase ,       rstrip=UpperCamelCase  ) if isinstance(UpperCamelCase ,       UpperCamelCase  ) else eos_token
												__UpperCAmelCase   :						Optional[int]      							=					AddedToken(UpperCamelCase ,       lstrip=UpperCamelCase ,       rstrip=UpperCamelCase  ) if isinstance(UpperCamelCase ,       UpperCamelCase  ) else sep_token
												__UpperCAmelCase   :						List[Any]      							=					AddedToken(UpperCamelCase ,       lstrip=UpperCamelCase ,       rstrip=UpperCamelCase  ) if isinstance(UpperCamelCase ,       UpperCamelCase  ) else cls_token
												__UpperCAmelCase   :						List[Any]      							=					AddedToken(UpperCamelCase ,       lstrip=UpperCamelCase ,       rstrip=UpperCamelCase  ) if isinstance(UpperCamelCase ,       UpperCamelCase  ) else unk_token
												__UpperCAmelCase   :						int      							=					AddedToken(UpperCamelCase ,       lstrip=UpperCamelCase ,       rstrip=UpperCamelCase  ) if isinstance(UpperCamelCase ,       UpperCamelCase  ) else pad_token

												# Mask token behave like a normal word, i.e. include the space before it
												__UpperCAmelCase   :						Any      							=					AddedToken(UpperCamelCase ,       lstrip=UpperCamelCase ,       rstrip=UpperCamelCase  ) if isinstance(UpperCamelCase ,       UpperCamelCase  ) else mask_token

												super().__init__(
												    errors=UpperCamelCase ,       bos_token=UpperCamelCase ,       eos_token=UpperCamelCase ,       unk_token=UpperCamelCase ,       sep_token=UpperCamelCase ,       cls_token=UpperCamelCase ,       pad_token=UpperCamelCase ,       mask_token=UpperCamelCase ,       add_prefix_space=UpperCamelCase ,       **UpperCamelCase ,       )

												with open(UpperCamelCase ,       encoding="""utf-8"""  ) as vocab_handle:
																			__UpperCAmelCase   :						Dict      							=					json.load(UpperCamelCase  )
												__UpperCAmelCase   :						List[str]      							=					{v: k for k, v in self.encoder.items()}
												__UpperCAmelCase   :						List[str]      							=					errors  # how to handle errors in decoding
												__UpperCAmelCase   :						Optional[int]      							=					bytes_to_unicode()
												__UpperCAmelCase   :						Optional[Any]      							=					{v: k for k, v in self.byte_encoder.items()}
												with open(UpperCamelCase ,       encoding="""utf-8"""  ) as merges_handle:
																			__UpperCAmelCase   :						Dict      							=					merges_handle.read().split("""\n"""  )[1:-1]
												__UpperCAmelCase   :						List[str]      							=					[tuple(merge.split()  ) for merge in bpe_merges]
												__UpperCAmelCase   :						List[Any]      							=					dict(zip(UpperCamelCase ,       range(len(UpperCamelCase  )  )  )  )
												__UpperCAmelCase   :						str      							=					{}
												__UpperCAmelCase   :						Optional[int]      							=					add_prefix_space

												# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
												__UpperCAmelCase   :						str      							=					re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""  )






					@property
					def lowerCamelCase__							(       self    : Dict  ):
												'''simple docstring'''






												return len(self.encoder  )






					def lowerCamelCase__							(       self    : Tuple  ):
												'''simple docstring'''






												return dict(self.encoder ,       **self.added_tokens_encoder  )






					def lowerCamelCase__							(       self    : str ,       UpperCamelCase    : Tuple  ):
												'''simple docstring'''






												if token in self.cache:
																			return self.cache[token]
												__UpperCAmelCase   :						Optional[Any]      							=					tuple(UpperCamelCase  )
												__UpperCAmelCase   :						str      							=					get_pairs(UpperCamelCase  )

												if not pairs:
																			return token

												while True:
																			__UpperCAmelCase   :						List[Any]      							=					min(UpperCamelCase ,       key=lambda UpperCamelCase  : self.bpe_ranks.get(UpperCamelCase ,       float("""inf"""  )  )  )
																			if bigram not in self.bpe_ranks:
																										break
																			__UpperCAmelCase							,__UpperCAmelCase   :						Union[str, Any]      							=					bigram
																			__UpperCAmelCase   :						str      							=					[]
																			__UpperCAmelCase   :						int      							=					0
																			while i < len(UpperCamelCase  ):
																										try:
																																	__UpperCAmelCase   :						Optional[Any]      							=					word.index(UpperCamelCase ,       UpperCamelCase  )
																										except ValueError:
																																	new_word.extend(word[i:]  )
																																	break
																										else:
																																	new_word.extend(word[i:j]  )
																																	__UpperCAmelCase   :						Tuple      							=					j

																										if word[i] == first and i < len(UpperCamelCase  ) - 1 and word[i + 1] == second:
																																	new_word.append(first + second  )
																																	i += 2
																										else:
																																	new_word.append(word[i]  )
																																	i += 1
																			__UpperCAmelCase   :						Dict      							=					tuple(UpperCamelCase  )
																			__UpperCAmelCase   :						Dict      							=					new_word
																			if len(UpperCamelCase  ) == 1:
																										break
																			else:
																										__UpperCAmelCase   :						List[str]      							=					get_pairs(UpperCamelCase  )
												__UpperCAmelCase   :						Dict      							=					""" """.join(UpperCamelCase  )
												__UpperCAmelCase   :						int      							=					word
												return word






					def lowerCamelCase__							(       self    : Union[str, Any] ,       UpperCamelCase    : int  ):
												'''simple docstring'''






												__UpperCAmelCase   :						str      							=					[]
												for token in re.findall(self.pat ,       UpperCamelCase  ):
																			__UpperCAmelCase   :						Union[str, Any]      							=					"""""".join(
																			    self.byte_encoder[b] for b in token.encode("""utf-8"""  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																			bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase  ).split(""" """  )  )
												return bpe_tokens






					def lowerCamelCase__							(       self    : List[Any] ,       UpperCamelCase    : int  ):
												'''simple docstring'''






												return self.encoder.get(UpperCamelCase ,       self.encoder.get(self.unk_token  )  )






					def lowerCamelCase__							(       self    : int ,       UpperCamelCase    : Tuple  ):
												'''simple docstring'''






												return self.decoder.get(UpperCamelCase  )






					def lowerCamelCase__							(       self    : Tuple ,       UpperCamelCase    : Union[str, Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					"""""".join(UpperCamelCase  )
												__UpperCAmelCase   :						str      							=					bytearray([self.byte_decoder[c] for c in text]  ).decode("""utf-8""" ,       errors=self.errors  )
												return text






					def lowerCamelCase__							(       self    : Any ,       UpperCamelCase    : str ,       UpperCamelCase    : Optional[str] = None  ):
												'''simple docstring'''






												if not os.path.isdir(UpperCamelCase  ):
																			logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
																			return
												__UpperCAmelCase   :						Any      							=					os.path.join(
												    UpperCamelCase ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
												__UpperCAmelCase   :						List[str]      							=					os.path.join(
												    UpperCamelCase ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]  )

												with open(UpperCamelCase ,       """w""" ,       encoding="""utf-8"""  ) as f:
																			f.write(json.dumps(self.encoder ,       indent=2 ,       sort_keys=UpperCamelCase ,       ensure_ascii=UpperCamelCase  ) + """\n"""  )

												__UpperCAmelCase   :						Optional[Any]      							=					0
												with open(UpperCamelCase ,       """w""" ,       encoding="""utf-8"""  ) as writer:
																			writer.write("""#version: 0.2\n"""  )
																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,       key=lambda UpperCamelCase  : kv[1]  ):
																										if index != token_index:
																																	logger.warning(
																																	    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																																	    """ Please check that the tokenizer is not corrupted!"""  )
																																	__UpperCAmelCase   :						Dict      							=					token_index
																										writer.write(""" """.join(UpperCamelCase  ) + """\n"""  )
																										index += 1

												return vocab_file, merge_file






					def lowerCamelCase__							(       self    : Optional[int] ,       UpperCamelCase    : List[int] ,       UpperCamelCase    : Optional[List[int]] = None  ):
												'''simple docstring'''






												if token_ids_a is None:
																			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
												__UpperCAmelCase   :						Tuple      							=					[self.cls_token_id]
												__UpperCAmelCase   :						int      							=					[self.sep_token_id]
												return cls + token_ids_a + sep + sep + token_ids_a + sep






					def lowerCamelCase__							(       self    : Dict ,       UpperCamelCase    : List[int] ,       UpperCamelCase    : Optional[List[int]] = None ,       UpperCamelCase    : bool = False  ):
												'''simple docstring'''






												if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=UpperCamelCase ,       token_ids_a=UpperCamelCase ,       already_has_special_tokens=UpperCamelCase  )

												if token_ids_a is None:
																			return [1] + ([0] * len(UpperCamelCase  )) + [1]
												return [1] + ([0] * len(UpperCamelCase  )) + [1, 1] + ([0] * len(UpperCamelCase  )) + [1]






					def lowerCamelCase__							(       self    : Union[str, Any] ,       UpperCamelCase    : List[int] ,       UpperCamelCase    : Optional[List[int]] = None  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Union[str, Any]      							=					[self.sep_token_id]
												__UpperCAmelCase   :						int      							=					[self.cls_token_id]

												if token_ids_a is None:
																			return len(cls + token_ids_a + sep  ) * [0]
												return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]






					def lowerCamelCase__							(       self    : Optional[Any] ,       UpperCamelCase    : Union[str, Any] ,       UpperCamelCase    : Dict=False ,       **UpperCamelCase    : Optional[int]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					kwargs.pop("""add_prefix_space""" ,       self.add_prefix_space  )
												if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase  ) > 0 and not text[0].isspace()):
																			__UpperCAmelCase   :						Dict      							=					""" """ + text
												return (text, kwargs)




"""simple docstring"""






from importlib import import_module

from .logging import get_logger


UpperCAmelCase    :     Any   						=		get_logger(__name__)





class      lowerCamelCase__   :







					"""simple docstring"""


					def __init__(       self    : Optional[int] ,       UpperCamelCase    : List[Any] ,       UpperCamelCase    : int=None  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					attrs or []
												if module is not None:
																			for key in module.__dict__:
																										if key in attrs or not key.startswith("""__"""  ):
																																	setattr(self ,       UpperCamelCase ,       getattr(UpperCamelCase ,       UpperCamelCase  )  )
												__UpperCAmelCase   :						Any      							=					module._original_module if isinstance(UpperCamelCase ,       _PatchedModuleObj  ) else module






class      lowerCamelCase__   :







					"""simple docstring"""


					__a	       =							[]






					def __init__(       self    : str ,       UpperCamelCase    : Tuple ,       UpperCamelCase    : str ,       UpperCamelCase    : List[str] ,       UpperCamelCase    : Optional[Any]=None  ):
												'''simple docstring'''






												__UpperCAmelCase   :						int      							=					obj
												__UpperCAmelCase   :						Union[str, Any]      							=					target
												__UpperCAmelCase   :						List[str]      							=					new
												__UpperCAmelCase   :						Optional[int]      							=					target.split("""."""  )[0]
												__UpperCAmelCase   :						Tuple      							=					{}
												__UpperCAmelCase   :						Union[str, Any]      							=					attrs or []






					def __enter__(       self    : Dict  ):
												'''simple docstring'''






												*__UpperCAmelCase							,__UpperCAmelCase   :						str      							=					self.target.split("""."""  )

												# Patch modules:
												# it's used to patch attributes of submodules like "os.path.join";
												# in this case we need to patch "os" and "os.path"

												for i in range(len(UpperCamelCase  )  ):
																			try:
																										__UpperCAmelCase   :						List[Any]      							=					import_module(""".""".join(submodules[: i + 1]  )  )
																			except ModuleNotFoundError:
																										continue
																			# We iterate over all the globals in self.obj in case we find "os" or "os.path"
																			for attr in self.obj.__dir__():
																										__UpperCAmelCase   :						List[Any]      							=					getattr(self.obj ,       UpperCamelCase  )
																										# We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
																										# This allows to patch renamed modules like "from os import path as ospath".
																										if obj_attr is submodule or (
																										    (isinstance(UpperCamelCase ,       _PatchedModuleObj  ) and obj_attr._original_module is submodule)
																										):
																																	__UpperCAmelCase   :						Tuple      							=					obj_attr
																																	# patch at top level
																																	setattr(self.obj ,       UpperCamelCase ,       _PatchedModuleObj(UpperCamelCase ,       attrs=self.attrs  )  )
																																	__UpperCAmelCase   :						int      							=					getattr(self.obj ,       UpperCamelCase  )
																																	# construct lower levels patches
																																	for key in submodules[i + 1 :]:
																																								setattr(UpperCamelCase ,       UpperCamelCase ,       _PatchedModuleObj(getattr(UpperCamelCase ,       UpperCamelCase ,       UpperCamelCase  ) ,       attrs=self.attrs  )  )
																																								__UpperCAmelCase   :						Optional[int]      							=					getattr(UpperCamelCase ,       UpperCamelCase  )
																																	# finally set the target attribute
																																	setattr(UpperCamelCase ,       UpperCamelCase ,       self.new  )

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

												if submodules:  # if it's an attribute of a submodule like "os.path.join"
																			try:
																										__UpperCAmelCase   :						int      							=					getattr(import_module(""".""".join(UpperCamelCase  )  ) ,       UpperCamelCase  )
																			except (AttributeError, ModuleNotFoundError):
																										return
																			# We iterate over all the globals in self.obj in case we find "os.path.join"
																			for attr in self.obj.__dir__():
																										# We don't check for the name of the global, but rather if its value *is* "os.path.join".
																										# This allows to patch renamed attributes like "from os.path import join as pjoin".
																										if getattr(self.obj ,       UpperCamelCase  ) is attr_value:
																																	__UpperCAmelCase   :						Union[str, Any]      							=					getattr(self.obj ,       UpperCamelCase  )
																																	setattr(self.obj ,       UpperCamelCase ,       self.new  )
												elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
																			__UpperCAmelCase   :						str      							=					globals()["""__builtins__"""][target_attr]
																			setattr(self.obj ,       UpperCamelCase ,       self.new  )
												else:
																			raise RuntimeError(f'''Tried to patch attribute {target_attr} instead of a submodule.'''  )






					def __exit__(       self    : str ,       *UpperCamelCase    : Optional[int]  ):
												'''simple docstring'''






												for attr in list(self.original  ):
																			setattr(self.obj ,       UpperCamelCase ,       self.original.pop(UpperCamelCase  )  )






					def lowerCamelCase__							(       self    : int  ):
												'''simple docstring'''






												self.__enter__()
												self._active_patches.append(self  )






					def lowerCamelCase__							(       self    : List[Any]  ):
												'''simple docstring'''






												try:
																			self._active_patches.remove(self  )
												except ValueError:
																			# If the patch hasn't been started this will fail
																			return None

												return self.__exit__()
