




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__    			=			logging.get_logger(__name__)


UpperCamelCase__    			=			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

UpperCamelCase__    			=			{
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
}

UpperCamelCase__    			=			{
    """allenai/longformer-base-4096""": 4096,
    """allenai/longformer-large-4096""": 4096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4096,
}







@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 	_a      (						):
					__lowerCAmelCase				   =       (
					    list(range(ord("!"     )      ,      ord("~"     ) + 1     )     ) + list(range(ord("¡"     )      ,      ord("¬"     ) + 1     )     ) + list(range(ord("®"     )      ,      ord("ÿ"     ) + 1     )     )
					)
					__lowerCAmelCase				   =       bs[:]
					__lowerCAmelCase				   =       0
					for b in range(2**8     ):
										if b not in bs:
															bs.append(SCREAMING_SNAKE_CASE_     )
															cs.append(2**8 + n     )
															n += 1
					__lowerCAmelCase				   =       [chr(SCREAMING_SNAKE_CASE_     ) for n in cs]
					return dict(zip(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )     )







def 	_a      (						SCREAMING_SNAKE_CASE_							:				List[Any]     ):
					__lowerCAmelCase				   =       set()
					__lowerCAmelCase				   =       word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)     )
										__lowerCAmelCase				   =       char
					return pairs


class 			a__    (					snake_case__      ):
						_a							:    str					=    VOCAB_FILES_NAMES
						_a							:    List[str]					=    PRETRAINED_VOCAB_FILES_MAP
						_a							:    List[str]					=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_a							:    Optional[Any]					=    ["""input_ids""", """attention_mask"""]






						def __init__(   self		,			_A		,			_A		,			_A="replace"		,			_A="<s>"		,			_A="</s>"		,			_A="</s>"		,			_A="<s>"		,			_A="<unk>"		,			_A="<pad>"		,			_A="<mask>"		,			_A=False		,			**_A		,			):




											"""simple docstring"""



											__lowerCAmelCase				   =       AddedToken(_A		,			lstrip=_A		,			rstrip=_A			) if isinstance(_A		,			_A			) else bos_token
											__lowerCAmelCase				   =       AddedToken(_A		,			lstrip=_A		,			rstrip=_A			) if isinstance(_A		,			_A			) else eos_token
											__lowerCAmelCase				   =       AddedToken(_A		,			lstrip=_A		,			rstrip=_A			) if isinstance(_A		,			_A			) else sep_token
											__lowerCAmelCase				   =       AddedToken(_A		,			lstrip=_A		,			rstrip=_A			) if isinstance(_A		,			_A			) else cls_token
											__lowerCAmelCase				   =       AddedToken(_A		,			lstrip=_A		,			rstrip=_A			) if isinstance(_A		,			_A			) else unk_token
											__lowerCAmelCase				   =       AddedToken(_A		,			lstrip=_A		,			rstrip=_A			) if isinstance(_A		,			_A			) else pad_token

											# Mask token behave like a normal word, i.e. include the space before it
											__lowerCAmelCase				   =       AddedToken(_A		,			lstrip=_A		,			rstrip=_A			) if isinstance(_A		,			_A			) else mask_token

											super().__init__(
											    errors=_A		,			bos_token=_A		,			eos_token=_A		,			unk_token=_A		,			sep_token=_A		,			cls_token=_A		,			pad_token=_A		,			mask_token=_A		,			add_prefix_space=_A		,			**_A		,			)

											with open(_A		,			encoding="utf-8"			) as vocab_handle:
																__lowerCAmelCase				   =       json.load(_A			)
											__lowerCAmelCase				   =       {v: k for k, v in self.encoder.items()}
											__lowerCAmelCase				   =       errors  # how to handle errors in decoding
											__lowerCAmelCase				   =       bytes_to_unicode()
											__lowerCAmelCase				   =       {v: k for k, v in self.byte_encoder.items()}
											with open(_A		,			encoding="utf-8"			) as merges_handle:
																__lowerCAmelCase				   =       merges_handle.read().split("\n"			)[1:-1]
											__lowerCAmelCase				   =       [tuple(merge.split()			) for merge in bpe_merges]
											__lowerCAmelCase				   =       dict(zip(_A		,			range(len(_A			)			)			)			)
											__lowerCAmelCase				   =       {}
											__lowerCAmelCase				   =       add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											__lowerCAmelCase				   =       re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"			)






						@property
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											return len(self.encoder			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											return dict(self.encoder		,			**self.added_tokens_encoder			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											if token in self.cache:
																return self.cache[token]
											__lowerCAmelCase				   =       tuple(_A			)
											__lowerCAmelCase				   =       get_pairs(_A			)

											if not pairs:
																return token

											while True:
																__lowerCAmelCase				   =       min(_A		,			key=lambda _A			: self.bpe_ranks.get(_A		,			float("inf"			)			)			)
																if bigram not in self.bpe_ranks:
																					break
																__lowerCAmelCase   ,							__lowerCAmelCase				   =       bigram
																__lowerCAmelCase				   =       []
																__lowerCAmelCase				   =       0
																while i < len(_A			):
																					try:
																										__lowerCAmelCase				   =       word.index(_A		,			_A			)
																					except ValueError:
																										new_word.extend(word[i:]			)
																										break
																					else:
																										new_word.extend(word[i:j]			)
																										__lowerCAmelCase				   =       j

																					if word[i] == first and i < len(_A			) - 1 and word[i + 1] == second:
																										new_word.append(first + second			)
																										i += 2
																					else:
																										new_word.append(word[i]			)
																										i += 1
																__lowerCAmelCase				   =       tuple(_A			)
																__lowerCAmelCase				   =       new_word
																if len(_A			) == 1:
																					break
																else:
																					__lowerCAmelCase				   =       get_pairs(_A			)
											__lowerCAmelCase				   =       " ".join(_A			)
											__lowerCAmelCase				   =       word
											return word






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       []
											for token in re.findall(self.pat		,			_A			):
																__lowerCAmelCase				   =       "".join(
																    self.byte_encoder[b] for b in token.encode("utf-8"			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_A			).split(" "			)			)
											return bpe_tokens






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											return self.encoder.get(_A		,			self.encoder.get(self.unk_token			)			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											return self.decoder.get(_A			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "".join(_A			)
											__lowerCAmelCase				   =       bytearray([self.byte_decoder[c] for c in text]			).decode("utf-8"		,			errors=self.errors			)
											return text






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A = None			):




											"""simple docstring"""



											if not os.path.isdir(_A			):
																logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""			)
																return
											__lowerCAmelCase				   =       os.path.join(
											    _A		,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
											__lowerCAmelCase				   =       os.path.join(
											    _A		,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

											with open(_A		,			"w"		,			encoding="utf-8"			) as f:
																f.write(json.dumps(self.encoder		,			indent=2		,			sort_keys=_A		,			ensure_ascii=_A			) + "\n"			)

											__lowerCAmelCase				   =       0
											with open(_A		,			"w"		,			encoding="utf-8"			) as writer:
																writer.write("#version: 0.2\n"			)
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,			key=lambda _A			: kv[1]			):
																					if index != token_index:
																										logger.warning(
																										    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																										    " Please check that the tokenizer is not corrupted!"			)
																										__lowerCAmelCase				   =       token_index
																					writer.write(" ".join(_A			) + "\n"			)
																					index += 1

											return vocab_file, merge_file






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A = None			):




											"""simple docstring"""



											if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											__lowerCAmelCase				   =       [self.cls_token_id]
											__lowerCAmelCase				   =       [self.sep_token_id]
											return cls + token_ids_a + sep + sep + token_ids_a + sep






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A = None		,			_A = False			):




											"""simple docstring"""



											if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_A		,			token_ids_a=_A		,			already_has_special_tokens=_A			)

											if token_ids_a is None:
																return [1] + ([0] * len(_A			)) + [1]
											return [1] + ([0] * len(_A			)) + [1, 1] + ([0] * len(_A			)) + [1]






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A = None			):




											"""simple docstring"""



											__lowerCAmelCase				   =       [self.sep_token_id]
											__lowerCAmelCase				   =       [self.cls_token_id]

											if token_ids_a is None:
																return len(cls + token_ids_a + sep			) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A=False		,			**_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       kwargs.pop("add_prefix_space"		,			self.add_prefix_space			)
											if (is_split_into_words or add_prefix_space) and (len(_A			) > 0 and not text[0].isspace()):
																__lowerCAmelCase				   =       " " + text
											return (text, kwargs)










from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
						if not (is_transformers_available() and is_torch_available() and is_transformers_version('''>=''', '''4.25.0''')):
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline
else:
						from .pipeline_unclip import UnCLIPPipeline
						from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline
						from .text_proj import UnCLIPTextProjModel




