



import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__   			=   logging.get_logger(__name__)


SCREAMING_SNAKE_CASE__   			=   {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all BART models at https://huggingface.co/models?filter=bart
SCREAMING_SNAKE_CASE__   			=   {
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
}

SCREAMING_SNAKE_CASE__   			=   {
    '''facebook/bart-base''': 1_0_2_4,
    '''facebook/bart-large''': 1_0_2_4,
    '''facebook/bart-large-mnli''': 1_0_2_4,
    '''facebook/bart-large-cnn''': 1_0_2_4,
    '''facebook/bart-large-xsum''': 1_0_2_4,
    '''yjernite/bart_eli5''': 1_0_2_4,
}


@lru_cache()
def A    (							)    ->   List[str]:
			A__	  =     (
			    list(range(ord('!'       )			,						ord('~'       ) + 1       )       ) + list(range(ord('¡'       )			,						ord('¬'       ) + 1       )       ) + list(range(ord('®'       )			,						ord('ÿ'       ) + 1       )       )
			)
			A__	  =     bs[:]
			A__	  =     0
			for b in range(2**8       ):
						if b not in bs:
									bs.append(__UpperCamelCase       )
									cs.append(2**8 + n       )
									n += 1
			A__	  =     [chr(__UpperCamelCase       ) for n in cs]
			return dict(zip(__UpperCamelCase			,						__UpperCamelCase       )       )


def A    (							__UpperCamelCase       )    ->   Union[str, Any]:
			A__	  =     set()
			A__	  =     word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)       )
						A__	  =     char
			return pairs






class   __lowerCAmelCase  (		UpperCAmelCase_   ):
					"""simple docstring"""






					A__					:		Union[str, Any]      							=     VOCAB_FILES_NAMES
					A__					:		Optional[Any]      							=     PRETRAINED_VOCAB_FILES_MAP
					A__					:		Any      							=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A__					:		Tuple      							=     ["input_ids", "attention_mask"]



					def __init__(  self       :							Any					,       _snake_case       :							List[Any]					,       _snake_case       :							str					,       _snake_case       :							Optional[int]="replace"					,       _snake_case       :							Optional[int]="<s>"					,       _snake_case       :							int="</s>"					,       _snake_case       :							Optional[int]="</s>"					,       _snake_case       :							Any="<s>"					,       _snake_case       :							int="<unk>"					,       _snake_case       :							str="<pad>"					,       _snake_case       :							List[Any]="<mask>"					,       _snake_case       :							Dict=False					,       **_snake_case       :							List[Any]					,       ):






								"""simple docstring"""


								A__	  =     AddedToken(_snake_case					,       lstrip=_snake_case					,       rstrip=_snake_case       ) if isinstance(_snake_case					,       _snake_case       ) else bos_token
								A__	  =     AddedToken(_snake_case					,       lstrip=_snake_case					,       rstrip=_snake_case       ) if isinstance(_snake_case					,       _snake_case       ) else eos_token
								A__	  =     AddedToken(_snake_case					,       lstrip=_snake_case					,       rstrip=_snake_case       ) if isinstance(_snake_case					,       _snake_case       ) else sep_token
								A__	  =     AddedToken(_snake_case					,       lstrip=_snake_case					,       rstrip=_snake_case       ) if isinstance(_snake_case					,       _snake_case       ) else cls_token
								A__	  =     AddedToken(_snake_case					,       lstrip=_snake_case					,       rstrip=_snake_case       ) if isinstance(_snake_case					,       _snake_case       ) else unk_token
								A__	  =     AddedToken(_snake_case					,       lstrip=_snake_case					,       rstrip=_snake_case       ) if isinstance(_snake_case					,       _snake_case       ) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								A__	  =     AddedToken(_snake_case					,       lstrip=_snake_case					,       rstrip=_snake_case       ) if isinstance(_snake_case					,       _snake_case       ) else mask_token

								super().__init__(
								    errors=_snake_case					,       bos_token=_snake_case					,       eos_token=_snake_case					,       unk_token=_snake_case					,       sep_token=_snake_case					,       cls_token=_snake_case					,       pad_token=_snake_case					,       mask_token=_snake_case					,       add_prefix_space=_snake_case					,       **_snake_case					,       )

								with open(_snake_case					,       encoding='utf-8'       ) as vocab_handle:
											A__	  =     json.load(_snake_case       )
								A__	  =     {v: k for k, v in self.encoder.items()}
								A__	  =     errors  # how to handle errors in decoding
								A__	  =     bytes_to_unicode()
								A__	  =     {v: k for k, v in self.byte_encoder.items()}
								with open(_snake_case					,       encoding='utf-8'       ) as merges_handle:
											A__	  =     merges_handle.read().split('\n'       )[1:-1]
								A__	  =     [tuple(merge.split()       ) for merge in bpe_merges]
								A__	  =     dict(zip(_snake_case					,       range(len(_snake_case       )       )       )       )
								A__	  =     {}
								A__	  =     add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								A__	  =     re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'       )



					@property
					def      _a				(  self       :							List[Any]       ):






								"""simple docstring"""


								return len(self.encoder       )



					def      _a				(  self       :							List[Any]       ):






								"""simple docstring"""


								return dict(self.encoder					,       **self.added_tokens_encoder       )



					def      _a				(  self       :							Tuple					,       _snake_case       :							Tuple       ):






								"""simple docstring"""


								if token in self.cache:
											return self.cache[token]
								A__	  =     tuple(_snake_case       )
								A__	  =     get_pairs(_snake_case       )

								if not pairs:
											return token

								while True:
											A__	  =     min(_snake_case					,       key=lambda _snake_case       : self.bpe_ranks.get(_snake_case					,       float('inf'       )       )       )
											if bigram not in self.bpe_ranks:
														break
											A__  ,			A__	  =     bigram
											A__	  =     []
											A__	  =     0
											while i < len(_snake_case       ):
														try:
																	A__	  =     word.index(_snake_case					,       _snake_case       )
														except ValueError:
																	new_word.extend(word[i:]       )
																	break
														else:
																	new_word.extend(word[i:j]       )
																	A__	  =     j

														if word[i] == first and i < len(_snake_case       ) - 1 and word[i + 1] == second:
																	new_word.append(first + second       )
																	i += 2
														else:
																	new_word.append(word[i]       )
																	i += 1
											A__	  =     tuple(_snake_case       )
											A__	  =     new_word
											if len(_snake_case       ) == 1:
														break
											else:
														A__	  =     get_pairs(_snake_case       )
								A__	  =     ' '.join(_snake_case       )
								A__	  =     word
								return word



					def      _a				(  self       :							Dict					,       _snake_case       :							str       ):






								"""simple docstring"""


								A__	  =     []
								for token in re.findall(self.pat					,       _snake_case       ):
											A__	  =     ''.join(
											    self.byte_encoder[b] for b in token.encode('utf-8'       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_snake_case       ).split(' '       )       )
								return bpe_tokens



					def      _a				(  self       :							Union[str, Any]					,       _snake_case       :							Union[str, Any]       ):






								"""simple docstring"""


								return self.encoder.get(_snake_case					,       self.encoder.get(self.unk_token       )       )



					def      _a				(  self       :							Tuple					,       _snake_case       :							List[Any]       ):






								"""simple docstring"""


								return self.decoder.get(_snake_case       )



					def      _a				(  self       :							Optional[Any]					,       _snake_case       :							Dict       ):






								"""simple docstring"""


								A__	  =     ''.join(_snake_case       )
								A__	  =     bytearray([self.byte_decoder[c] for c in text]       ).decode('utf-8'					,       errors=self.errors       )
								return text



					def      _a				(  self       :							List[str]					,       _snake_case       :							str					,       _snake_case       :							Optional[str] = None       ):






								"""simple docstring"""


								if not os.path.isdir(_snake_case       ):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
											return
								A__	  =     os.path.join(
								    _snake_case					,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )
								A__	  =     os.path.join(
								    _snake_case					,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']       )

								with open(_snake_case					,       'w'					,       encoding='utf-8'       ) as f:
											f.write(json.dumps(self.encoder					,       indent=2					,       sort_keys=_snake_case					,       ensure_ascii=_snake_case       ) + '\n'       )

								A__	  =     0
								with open(_snake_case					,       'w'					,       encoding='utf-8'       ) as writer:
											writer.write('#version: 0.2\n'       )
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,       key=lambda _snake_case       : kv[1]       ):
														if index != token_index:
																	logger.warning(
																	    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																	    ' Please check that the tokenizer is not corrupted!'       )
																	A__	  =     token_index
														writer.write(' '.join(_snake_case       ) + '\n'       )
														index += 1

								return vocab_file, merge_file



					def      _a				(  self       :							Tuple					,       _snake_case       :							List[int]					,       _snake_case       :							Optional[List[int]] = None       ):






								"""simple docstring"""


								if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								A__	  =     [self.cls_token_id]
								A__	  =     [self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep



					def      _a				(  self       :							Tuple					,       _snake_case       :							List[int]					,       _snake_case       :							Optional[List[int]] = None					,       _snake_case       :							bool = False       ):






								"""simple docstring"""


								if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=_snake_case					,       token_ids_a=_snake_case					,       already_has_special_tokens=_snake_case       )

								if token_ids_a is None:
											return [1] + ([0] * len(_snake_case       )) + [1]
								return [1] + ([0] * len(_snake_case       )) + [1, 1] + ([0] * len(_snake_case       )) + [1]



					def      _a				(  self       :							Optional[Any]					,       _snake_case       :							List[int]					,       _snake_case       :							Optional[List[int]] = None       ):






								"""simple docstring"""


								A__	  =     [self.sep_token_id]
								A__	  =     [self.cls_token_id]

								if token_ids_a is None:
											return len(cls + token_ids_a + sep       ) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]



					def      _a				(  self       :							int					,       _snake_case       :							Union[str, Any]					,       _snake_case       :							Tuple=False					,       **_snake_case       :							Tuple       ):






								"""simple docstring"""


								A__	  =     kwargs.pop('add_prefix_space'					,       self.add_prefix_space       )
								if (is_split_into_words or add_prefix_space) and (len(_snake_case       ) > 0 and not text[0].isspace()):
											A__	  =     ' ' + text
								return (text, kwargs)








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


_lowercase							:   int						=						{
    "configuration_roberta_prelayernorm": [
        "ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "RobertaPreLayerNormConfig",
        "RobertaPreLayerNormOnnxConfig",
    ],
}

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_lowercase							:   List[Any]						=						[
				    "ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "RobertaPreLayerNormForCausalLM",
				    "RobertaPreLayerNormForMaskedLM",
				    "RobertaPreLayerNormForMultipleChoice",
				    "RobertaPreLayerNormForQuestionAnswering",
				    "RobertaPreLayerNormForSequenceClassification",
				    "RobertaPreLayerNormForTokenClassification",
				    "RobertaPreLayerNormModel",
				    "RobertaPreLayerNormPreTrainedModel",
				]

try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_lowercase							:   Optional[int]						=						[
				    "TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "TFRobertaPreLayerNormForCausalLM",
				    "TFRobertaPreLayerNormForMaskedLM",
				    "TFRobertaPreLayerNormForMultipleChoice",
				    "TFRobertaPreLayerNormForQuestionAnswering",
				    "TFRobertaPreLayerNormForSequenceClassification",
				    "TFRobertaPreLayerNormForTokenClassification",
				    "TFRobertaPreLayerNormMainLayer",
				    "TFRobertaPreLayerNormModel",
				    "TFRobertaPreLayerNormPreTrainedModel",
				]

try:
				if not is_flax_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_lowercase							:   str						=						[
				    "FlaxRobertaPreLayerNormForCausalLM",
				    "FlaxRobertaPreLayerNormForMaskedLM",
				    "FlaxRobertaPreLayerNormForMultipleChoice",
				    "FlaxRobertaPreLayerNormForQuestionAnswering",
				    "FlaxRobertaPreLayerNormForSequenceClassification",
				    "FlaxRobertaPreLayerNormForTokenClassification",
				    "FlaxRobertaPreLayerNormModel",
				    "FlaxRobertaPreLayerNormPreTrainedModel",
				]


if TYPE_CHECKING:
				from .configuration_roberta_prelayernorm import (
				    ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP,
				    RobertaPreLayerNormConfig,
				    RobertaPreLayerNormOnnxConfig,
				)

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_roberta_prelayernorm import (
								    ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
								    RobertaPreLayerNormForCausalLM,
								    RobertaPreLayerNormForMaskedLM,
								    RobertaPreLayerNormForMultipleChoice,
								    RobertaPreLayerNormForQuestionAnswering,
								    RobertaPreLayerNormForSequenceClassification,
								    RobertaPreLayerNormForTokenClassification,
								    RobertaPreLayerNormModel,
								    RobertaPreLayerNormPreTrainedModel,
								)

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_roberta_prelayernorm import (
								    TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFRobertaPreLayerNormForCausalLM,
								    TFRobertaPreLayerNormForMaskedLM,
								    TFRobertaPreLayerNormForMultipleChoice,
								    TFRobertaPreLayerNormForQuestionAnswering,
								    TFRobertaPreLayerNormForSequenceClassification,
								    TFRobertaPreLayerNormForTokenClassification,
								    TFRobertaPreLayerNormMainLayer,
								    TFRobertaPreLayerNormModel,
								    TFRobertaPreLayerNormPreTrainedModel,
								)

				try:
								if not is_flax_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_flax_roberta_prelayernorm import (
								    FlaxRobertaPreLayerNormForCausalLM,
								    FlaxRobertaPreLayerNormForMaskedLM,
								    FlaxRobertaPreLayerNormForMultipleChoice,
								    FlaxRobertaPreLayerNormForQuestionAnswering,
								    FlaxRobertaPreLayerNormForSequenceClassification,
								    FlaxRobertaPreLayerNormForTokenClassification,
								    FlaxRobertaPreLayerNormModel,
								    FlaxRobertaPreLayerNormPreTrainedModel,
								)

else:
				import sys

				_lowercase							:   List[Any]						=						_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



