





import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__           =			logging.get_logger(__name__)

class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   Optional[Any]      =							"""encoder-decoder"""
							snake_case  :   Optional[int]      =							True






							def __init__(    self	,			**__lowerCAmelCase	):
														super().__init__(**__lowerCAmelCase	)
														assert (
														    "encoder" in kwargs and "decoder" in kwargs
														), "Config has to be initialized with encoder and decoder config"
														UpperCamelCase__					  =      kwargs.pop("""encoder"""	)
														UpperCamelCase__					  =      encoder_config.pop("""model_type"""	)
														UpperCamelCase__					  =      kwargs.pop("""decoder"""	)
														UpperCamelCase__					  =      decoder_config.pop("""model_type"""	)

														from ..auto.configuration_auto import AutoConfig

														UpperCamelCase__					  =      AutoConfig.for_model(__lowerCAmelCase	,			**__lowerCAmelCase	)
														UpperCamelCase__					  =      AutoConfig.for_model(__lowerCAmelCase	,			**__lowerCAmelCase	)
														UpperCamelCase__					  =      True






							@classmethod
							def 					_lowerCamelCase		(    cls	,			__lowerCAmelCase	,			__lowerCAmelCase	,			**__lowerCAmelCase	):
														logger.info("""Set `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config"""	)
														UpperCamelCase__					  =      True
														UpperCamelCase__					  =      True

														return cls(encoder=encoder_config.to_dict()	,			decoder=decoder_config.to_dict()	,			**__lowerCAmelCase	)







							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      copy.deepcopy(self.__dict__	)
														UpperCamelCase__					  =      self.encoder.to_dict()
														UpperCamelCase__					  =      self.decoder.to_dict()
														UpperCamelCase__					  =      self.__class__.model_type
														return output











import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_big_bird import BigBirdTokenizer
else:
					UpperCamelCase__           =			None

UpperCamelCase__           =			logging.get_logger(__name__)
UpperCamelCase__           =			{"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

UpperCamelCase__           =			{
    "vocab_file": {
        "google/bigbird-roberta-base": "https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model",
        "google/bigbird-roberta-large": (
            "https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"
        ),
        "google/bigbird-base-trivia-itc": (
            "https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"
        ),
    },
    "tokenizer_file": {
        "google/bigbird-roberta-base": (
            "https://huggingface.co/google/bigbird-roberta-base/resolve/main/tokenizer.json"
        ),
        "google/bigbird-roberta-large": (
            "https://huggingface.co/google/bigbird-roberta-large/resolve/main/tokenizer.json"
        ),
        "google/bigbird-base-trivia-itc": (
            "https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/tokenizer.json"
        ),
    },
}

UpperCamelCase__           =			{
    "google/bigbird-roberta-base": 4_096,
    "google/bigbird-roberta-large": 4_096,
    "google/bigbird-base-trivia-itc": 4_096,
}


UpperCamelCase__           =			"▁"

class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   Optional[Any]      =							VOCAB_FILES_NAMES
							snake_case  :   List[str]      =							PRETRAINED_VOCAB_FILES_MAP
							snake_case  :   List[str]      =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case  :   int      =							BigBirdTokenizer
							snake_case  :   List[Any]      =							["""input_ids""", """attention_mask"""]
							snake_case  :   List[int]      =							[]






							def __init__(    self	,			__lowerCAmelCase=None	,			__lowerCAmelCase=None	,			__lowerCAmelCase="<unk>"	,			__lowerCAmelCase="<s>"	,			__lowerCAmelCase="</s>"	,			__lowerCAmelCase="<pad>"	,			__lowerCAmelCase="[SEP]"	,			__lowerCAmelCase="[MASK]"	,			__lowerCAmelCase="[CLS]"	,			**__lowerCAmelCase	,			):
														UpperCamelCase__					  =      AddedToken(__lowerCAmelCase	,			lstrip=__lowerCAmelCase	,			rstrip=__lowerCAmelCase	) if isinstance(__lowerCAmelCase	,			__lowerCAmelCase	) else bos_token
														UpperCamelCase__					  =      AddedToken(__lowerCAmelCase	,			lstrip=__lowerCAmelCase	,			rstrip=__lowerCAmelCase	) if isinstance(__lowerCAmelCase	,			__lowerCAmelCase	) else eos_token
														UpperCamelCase__					  =      AddedToken(__lowerCAmelCase	,			lstrip=__lowerCAmelCase	,			rstrip=__lowerCAmelCase	) if isinstance(__lowerCAmelCase	,			__lowerCAmelCase	) else unk_token
														UpperCamelCase__					  =      AddedToken(__lowerCAmelCase	,			lstrip=__lowerCAmelCase	,			rstrip=__lowerCAmelCase	) if isinstance(__lowerCAmelCase	,			__lowerCAmelCase	) else pad_token
														UpperCamelCase__					  =      AddedToken(__lowerCAmelCase	,			lstrip=__lowerCAmelCase	,			rstrip=__lowerCAmelCase	) if isinstance(__lowerCAmelCase	,			__lowerCAmelCase	) else cls_token
														UpperCamelCase__					  =      AddedToken(__lowerCAmelCase	,			lstrip=__lowerCAmelCase	,			rstrip=__lowerCAmelCase	) if isinstance(__lowerCAmelCase	,			__lowerCAmelCase	) else sep_token

														# Mask token behave like a normal word, i.e. include the space before it
														UpperCamelCase__					  =      AddedToken(__lowerCAmelCase	,			lstrip=__lowerCAmelCase	,			rstrip=__lowerCAmelCase	) if isinstance(__lowerCAmelCase	,			__lowerCAmelCase	) else mask_token

														super().__init__(
														    __lowerCAmelCase	,			tokenizer_file=__lowerCAmelCase	,			bos_token=__lowerCAmelCase	,			eos_token=__lowerCAmelCase	,			unk_token=__lowerCAmelCase	,			sep_token=__lowerCAmelCase	,			pad_token=__lowerCAmelCase	,			cls_token=__lowerCAmelCase	,			mask_token=__lowerCAmelCase	,			**__lowerCAmelCase	,			)

														UpperCamelCase__					  =      vocab_file
														UpperCamelCase__					  =      False if not self.vocab_file else True






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														UpperCamelCase__					  =      [self.sep_token_id]
														UpperCamelCase__					  =      [self.cls_token_id]
														if token_ids_a is None:
																					return cls + token_ids_a + sep
														return cls + token_ids_a + sep + token_ids_a + sep






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	,			__lowerCAmelCase = False	):

														if already_has_special_tokens:
																					if token_ids_a is not None:
																												raise ValueError(
																												    """You should not supply a second sequence if the provided sequence of """
																												    """ids is already formatted with special tokens for the model."""	)
																					return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

														if token_ids_a is None:
																					return [1] + ([0] * len(__lowerCAmelCase	)) + [1]
														return [1] + ([0] * len(__lowerCAmelCase	)) + [1] + ([0] * len(__lowerCAmelCase	)) + [1]






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														UpperCamelCase__					  =      [self.sep_token_id]
														UpperCamelCase__					  =      [self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep	) * [0]
														return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]







							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														if not self.can_save_slow_tokenizer:
																					raise ValueError(
																					    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
																					    """tokenizer."""	)

														if not os.path.isdir(__lowerCAmelCase	):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""	)
																					return
														UpperCamelCase__					  =      os.path.join(
														    __lowerCAmelCase	,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)

														if os.path.abspath(self.vocab_file	) != os.path.abspath(__lowerCAmelCase	):
																					copyfile(self.vocab_file	,			__lowerCAmelCase	)

														return (out_vocab_file,)




