





import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    FlaubertForMultipleChoice,
	    FlaubertForQuestionAnswering,
	    FlaubertForQuestionAnsweringSimple,
	    FlaubertForSequenceClassification,
	    FlaubertForTokenClassification,
	    FlaubertModel,
	    FlaubertWithLMHeadModel,
	)
	from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST




class 	UpperCamelCase_							(  UpperCamelCase__     ):


							def __init__(				self     :Union[str, Any]				,					__A     :Optional[int]				,					__A     :Tuple=13				,					__A     :Dict=7				,					__A     :Dict=True				,					__A     :str=True				,					__A     :Optional[Any]=True				,					__A     :Optional[Any]=True				,					__A     :Optional[Any]=True				,					__A     :Any=False				,					__A     :Dict=False				,					__A     :Any=False				,					__A     :Tuple=2				,					__A     :Dict=99				,					__A     :Optional[Any]=0				,					__A     :List[str]=32				,					__A     :Optional[int]=5				,					__A     :Dict=4				,					__A     :List[str]=0.1				,					__A     :Union[str, Any]=0.1				,					__A     :Tuple=512				,					__A     :Any=12				,					__A     :Optional[int]=2				,					__A     :Union[str, Any]=0.0_2				,					__A     :Dict=3				,					__A     :Optional[int]=4				,					__A     :Any="last"				,					__A     :List[Any]=None				,					__A     :Any=None				,					) -> str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   parent
									SCREAMING_SNAKE_CASE__								=   batch_size
									SCREAMING_SNAKE_CASE__								=   seq_length
									SCREAMING_SNAKE_CASE__								=   is_training
									SCREAMING_SNAKE_CASE__								=   use_input_lengths
									SCREAMING_SNAKE_CASE__								=   use_token_type_ids
									SCREAMING_SNAKE_CASE__								=   use_labels
									SCREAMING_SNAKE_CASE__								=   gelu_activation
									SCREAMING_SNAKE_CASE__								=   sinusoidal_embeddings
									SCREAMING_SNAKE_CASE__								=   causal
									SCREAMING_SNAKE_CASE__								=   asm
									SCREAMING_SNAKE_CASE__								=   n_langs
									SCREAMING_SNAKE_CASE__								=   vocab_size
									SCREAMING_SNAKE_CASE__								=   n_special
									SCREAMING_SNAKE_CASE__								=   hidden_size
									SCREAMING_SNAKE_CASE__								=   num_hidden_layers
									SCREAMING_SNAKE_CASE__								=   num_attention_heads
									SCREAMING_SNAKE_CASE__								=   hidden_dropout_prob
									SCREAMING_SNAKE_CASE__								=   attention_probs_dropout_prob
									SCREAMING_SNAKE_CASE__								=   max_position_embeddings
									SCREAMING_SNAKE_CASE__								=   type_vocab_size
									SCREAMING_SNAKE_CASE__								=   type_sequence_label_size
									SCREAMING_SNAKE_CASE__								=   initializer_range
									SCREAMING_SNAKE_CASE__								=   num_labels
									SCREAMING_SNAKE_CASE__								=   num_choices
									SCREAMING_SNAKE_CASE__								=   summary_type
									SCREAMING_SNAKE_CASE__								=   use_proj
									SCREAMING_SNAKE_CASE__								=   scope


							def   _snake_case	(				self     :Optional[Any]		) -> List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   ids_tensor([self.batch_size, self.seq_length]				,					self.vocab_size		)
									SCREAMING_SNAKE_CASE__								=   random_attention_mask([self.batch_size, self.seq_length]		)

									SCREAMING_SNAKE_CASE__								=   None
									if self.use_input_lengths:
											SCREAMING_SNAKE_CASE__								=   (
											    ids_tensor([self.batch_size]				,					vocab_size=2		) + self.seq_length - 2
											)  # small variation of seq_length

									SCREAMING_SNAKE_CASE__								=   None
									if self.use_token_type_ids:
											SCREAMING_SNAKE_CASE__								=   ids_tensor([self.batch_size, self.seq_length]				,					self.n_langs		)

									SCREAMING_SNAKE_CASE__								=   None
									SCREAMING_SNAKE_CASE__								=   None
									SCREAMING_SNAKE_CASE__								=   None
									if self.use_labels:
											SCREAMING_SNAKE_CASE__								=   ids_tensor([self.batch_size]				,					self.type_sequence_label_size		)
											SCREAMING_SNAKE_CASE__								=   ids_tensor([self.batch_size, self.seq_length]				,					self.num_labels		)
											SCREAMING_SNAKE_CASE__								=   ids_tensor([self.batch_size]				,					2		).float()
											SCREAMING_SNAKE_CASE__								=   ids_tensor([self.batch_size]				,					self.num_choices		)

									SCREAMING_SNAKE_CASE__								=   self.get_config()

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_lengths,
									    sequence_labels,
									    token_labels,
									    is_impossible_labels,
									    choice_labels,
									    input_mask,
									)


							def   _snake_case	(				self     :List[str]		) -> Optional[int]:



									"""simple docstring"""



									return FlaubertConfig(
									    vocab_size=self.vocab_size				,					n_special=self.n_special				,					emb_dim=self.hidden_size				,					n_layers=self.num_hidden_layers				,					n_heads=self.num_attention_heads				,					dropout=self.hidden_dropout_prob				,					attention_dropout=self.attention_probs_dropout_prob				,					gelu_activation=self.gelu_activation				,					sinusoidal_embeddings=self.sinusoidal_embeddings				,					asm=self.asm				,					causal=self.causal				,					n_langs=self.n_langs				,					max_position_embeddings=self.max_position_embeddings				,					initializer_range=self.initializer_range				,					summary_type=self.summary_type				,					use_proj=self.use_proj				,					)


							def   _snake_case	(				self     :Tuple				,					__A     :str				,					__A     :int				,					__A     :Optional[int]				,					__A     :Any				,					__A     :Union[str, Any]				,					__A     :Optional[int]				,					__A     :Union[str, Any]				,					__A     :Union[str, Any]				,					__A     :str				,					) -> List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   FlaubertModel(config=__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__								=   model(__A				,					lengths=__A				,					langs=__A		)
									SCREAMING_SNAKE_CASE__								=   model(__A				,					langs=__A		)
									SCREAMING_SNAKE_CASE__								=   model(__A		)
									self.parent.assertEqual(result.last_hidden_state.shape				,					(self.batch_size, self.seq_length, self.hidden_size)		)


							def   _snake_case	(				self     :str				,					__A     :Any				,					__A     :str				,					__A     :Union[str, Any]				,					__A     :Optional[Any]				,					__A     :Optional[int]				,					__A     :Any				,					__A     :Union[str, Any]				,					__A     :Optional[Any]				,					__A     :Union[str, Any]				,					) -> Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   FlaubertWithLMHeadModel(__A		)
									model.to(__A		)
									model.eval()

									SCREAMING_SNAKE_CASE__								=   model(__A				,					token_type_ids=__A				,					labels=__A		)
									self.parent.assertEqual(result.loss.shape				,					()		)
									self.parent.assertEqual(result.logits.shape				,					(self.batch_size, self.seq_length, self.vocab_size)		)


							def   _snake_case	(				self     :Tuple				,					__A     :Union[str, Any]				,					__A     :Optional[Any]				,					__A     :Dict				,					__A     :Dict				,					__A     :Union[str, Any]				,					__A     :List[str]				,					__A     :Optional[int]				,					__A     :int				,					__A     :str				,					) -> int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   FlaubertForQuestionAnsweringSimple(__A		)
									model.to(__A		)
									model.eval()

									SCREAMING_SNAKE_CASE__								=   model(__A		)

									SCREAMING_SNAKE_CASE__								=   model(__A				,					start_positions=__A				,					end_positions=__A		)
									self.parent.assertEqual(result.start_logits.shape				,					(self.batch_size, self.seq_length)		)
									self.parent.assertEqual(result.end_logits.shape				,					(self.batch_size, self.seq_length)		)


							def   _snake_case	(				self     :List[str]				,					__A     :Any				,					__A     :int				,					__A     :Tuple				,					__A     :Optional[Any]				,					__A     :Tuple				,					__A     :Optional[int]				,					__A     :str				,					__A     :int				,					__A     :str				,					) -> Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   FlaubertForQuestionAnswering(__A		)
									model.to(__A		)
									model.eval()

									SCREAMING_SNAKE_CASE__								=   model(__A		)

									SCREAMING_SNAKE_CASE__								=   model(
									    __A				,					start_positions=__A				,					end_positions=__A				,					cls_index=__A				,					is_impossible=__A				,					p_mask=__A				,					)

									SCREAMING_SNAKE_CASE__								=   model(
									    __A				,					start_positions=__A				,					end_positions=__A				,					cls_index=__A				,					is_impossible=__A				,					)

									((SCREAMING_SNAKE_CASE__)						,    )								=   result_with_labels.to_tuple()

									SCREAMING_SNAKE_CASE__								=   model(__A				,					start_positions=__A				,					end_positions=__A		)

									((SCREAMING_SNAKE_CASE__)						,    )								=   result_with_labels.to_tuple()

									self.parent.assertEqual(result_with_labels.loss.shape				,					()		)
									self.parent.assertEqual(result.start_top_log_probs.shape				,					(self.batch_size, model.config.start_n_top)		)
									self.parent.assertEqual(result.start_top_index.shape				,					(self.batch_size, model.config.start_n_top)		)
									self.parent.assertEqual(
									    result.end_top_log_probs.shape				,					(self.batch_size, model.config.start_n_top * model.config.end_n_top)		)
									self.parent.assertEqual(
									    result.end_top_index.shape				,					(self.batch_size, model.config.start_n_top * model.config.end_n_top)		)
									self.parent.assertEqual(result.cls_logits.shape				,					(self.batch_size,)		)


							def   _snake_case	(				self     :Optional[int]				,					__A     :str				,					__A     :Optional[int]				,					__A     :Tuple				,					__A     :Dict				,					__A     :List[str]				,					__A     :Tuple				,					__A     :List[str]				,					__A     :Dict				,					__A     :List[str]				,					) -> Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   FlaubertForSequenceClassification(__A		)
									model.to(__A		)
									model.eval()

									SCREAMING_SNAKE_CASE__								=   model(__A		)
									SCREAMING_SNAKE_CASE__								=   model(__A				,					labels=__A		)

									self.parent.assertEqual(result.loss.shape				,					()		)
									self.parent.assertEqual(result.logits.shape				,					(self.batch_size, self.type_sequence_label_size)		)


							def   _snake_case	(				self     :Optional[Any]				,					__A     :Optional[Any]				,					__A     :Optional[Any]				,					__A     :List[str]				,					__A     :Optional[Any]				,					__A     :int				,					__A     :Tuple				,					__A     :Optional[int]				,					__A     :Union[str, Any]				,					__A     :Dict				,					) -> Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self.num_labels
									SCREAMING_SNAKE_CASE__								=   FlaubertForTokenClassification(__A		)
									model.to(__A		)
									model.eval()

									SCREAMING_SNAKE_CASE__								=   model(__A				,					attention_mask=__A				,					labels=__A		)
									self.parent.assertEqual(result.logits.shape				,					(self.batch_size, self.seq_length, self.num_labels)		)


							def   _snake_case	(				self     :str				,					__A     :Any				,					__A     :Tuple				,					__A     :List[str]				,					__A     :Tuple				,					__A     :Any				,					__A     :int				,					__A     :Dict				,					__A     :List[str]				,					__A     :Tuple				,					) -> Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self.num_choices
									SCREAMING_SNAKE_CASE__								=   FlaubertForMultipleChoice(config=__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__								=   input_ids.unsqueeze(1		).expand(-1				,					self.num_choices				,					-1		).contiguous()
									SCREAMING_SNAKE_CASE__								=   token_type_ids.unsqueeze(1		).expand(-1				,					self.num_choices				,					-1		).contiguous()
									SCREAMING_SNAKE_CASE__								=   input_mask.unsqueeze(1		).expand(-1				,					self.num_choices				,					-1		).contiguous()
									SCREAMING_SNAKE_CASE__								=   model(
									    __A				,					attention_mask=__A				,					token_type_ids=__A				,					labels=__A				,					)
									self.parent.assertEqual(result.logits.shape				,					(self.batch_size, self.num_choices)		)


							def   _snake_case	(				self     :Union[str, Any]		) -> Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self.prepare_config_and_inputs()
									(
									    (
									    SCREAMING_SNAKE_CASE__
									)						,    (
									    SCREAMING_SNAKE_CASE__
									)						,    (
									    SCREAMING_SNAKE_CASE__
									)						,    (
									    SCREAMING_SNAKE_CASE__
									)						,    (
									    SCREAMING_SNAKE_CASE__
									)						,    (
									    SCREAMING_SNAKE_CASE__
									)						,    (
									    SCREAMING_SNAKE_CASE__
									)						,    (
									    SCREAMING_SNAKE_CASE__
									)						,    (
									    SCREAMING_SNAKE_CASE__
									)						,    
									)								=   config_and_inputs
									SCREAMING_SNAKE_CASE__								=   {
									    """input_ids""": input_ids,
									    """token_type_ids""": token_type_ids,
									    """lengths""": input_lengths,
									    """attention_mask""": input_mask,
									}
									return config, inputs_dict




@require_torch
class 	UpperCamelCase_							(  UpperCamelCase__					,	UpperCamelCase__					,	unittest.TestCase     ):
							lowerCamelCase_							     =       (
							    (
							        FlaubertModel,
							        FlaubertWithLMHeadModel,
							        FlaubertForQuestionAnswering,
							        FlaubertForQuestionAnsweringSimple,
							        FlaubertForSequenceClassification,
							        FlaubertForTokenClassification,
							        FlaubertForMultipleChoice,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCamelCase_							     =       (
							    {
							        "feature-extraction": FlaubertModel,
							        "fill-mask": FlaubertWithLMHeadModel,
							        "question-answering": FlaubertForQuestionAnsweringSimple,
							        "text-classification": FlaubertForSequenceClassification,
							        "token-classification": FlaubertForTokenClassification,
							        "zero-shot": FlaubertForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)


							def   _snake_case	(				self     :Any				,					__A     :Optional[int]				,					__A     :Optional[int]				,					__A     :Dict				,					__A     :List[Any]				,					__A     :Tuple		) -> str:



									"""simple docstring"""



									if (
									    pipeline_test_casse_name == "QAPipelineTests"
									    and tokenizer_name is not None
									    and not tokenizer_name.endswith("""Fast"""		)
									):
											# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
											# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
											# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
											return True

									return False


							def   _snake_case	(				self     :Tuple				,					__A     :List[str]				,					__A     :Optional[int]				,					__A     :Dict=False		) -> List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   super()._prepare_for_class(__A				,					__A				,					return_labels=__A		)

									if return_labels:
											if model_class.__name__ == "FlaubertForQuestionAnswering":
													SCREAMING_SNAKE_CASE__								=   torch.zeros(
													    self.model_tester.batch_size				,					dtype=torch.long				,					device=__A		)
													SCREAMING_SNAKE_CASE__								=   torch.zeros(
													    self.model_tester.batch_size				,					dtype=torch.long				,					device=__A		)

									return inputs_dict


							def   _snake_case	(				self     :str		) -> Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   FlaubertModelTester(self		)
									SCREAMING_SNAKE_CASE__								=   ConfigTester(self				,					config_class=__A				,					emb_dim=37		)


							def   _snake_case	(				self     :int		) -> int:



									"""simple docstring"""



									self.config_tester.run_common_tests()


							def   _snake_case	(				self     :Optional[Any]		) -> List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_model(*__A		)


							def   _snake_case	(				self     :Tuple		) -> List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_lm_head(*__A		)


							def   _snake_case	(				self     :str		) -> Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_simple_qa(*__A		)


							def   _snake_case	(				self     :Union[str, Any]		) -> Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_qa(*__A		)


							def   _snake_case	(				self     :str		) -> Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_sequence_classif(*__A		)


							def   _snake_case	(				self     :Any		) -> Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_token_classif(*__A		)


							def   _snake_case	(				self     :Any		) -> Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_multiple_choice(*__A		)


							@slow
							def   _snake_case	(				self     :Union[str, Any]		) -> List[str]:



									"""simple docstring"""



									for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											SCREAMING_SNAKE_CASE__								=   FlaubertModel.from_pretrained(__A		)
											self.assertIsNotNone(__A		)


							@slow
							@require_torch_gpu
							def   _snake_case	(				self     :Tuple		) -> List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__						,    SCREAMING_SNAKE_CASE__								=   self.model_tester.prepare_config_and_inputs_for_common()
									for model_class in self.all_model_classes:
											# FlauBertForMultipleChoice behaves incorrectly in JIT environments.
											if model_class == FlaubertForMultipleChoice:
													return

											SCREAMING_SNAKE_CASE__								=   True
											SCREAMING_SNAKE_CASE__								=   model_class(config=__A		)

											SCREAMING_SNAKE_CASE__								=   self._prepare_for_class(__A				,					__A		)
											SCREAMING_SNAKE_CASE__								=   torch.jit.trace(
											    __A				,					(inputs_dict["""input_ids"""].to("""cpu"""		), inputs_dict["""attention_mask"""].to("""cpu"""		))		)

											with tempfile.TemporaryDirectory() as tmp:
													torch.jit.save(__A				,					os.path.join(__A				,					"""traced_model.pt"""		)		)
													SCREAMING_SNAKE_CASE__								=   torch.jit.load(os.path.join(__A				,					"""traced_model.pt"""		)				,					map_location=__A		)
													loaded(inputs_dict["""input_ids"""].to(__A		)				,					inputs_dict["""attention_mask"""].to(__A		)		)




@require_torch
class 	UpperCamelCase_							(  unittest.TestCase     ):


							@slow
							def   _snake_case	(				self     :Dict		) -> Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   FlaubertModel.from_pretrained("""flaubert/flaubert_base_cased"""		)
									SCREAMING_SNAKE_CASE__								=   torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]		)
									with torch.no_grad():
											SCREAMING_SNAKE_CASE__								=   model(__A		)[0]
									SCREAMING_SNAKE_CASE__								=   torch.Size((1, 11, 768)		)
									self.assertEqual(output.shape				,					__A		)
									SCREAMING_SNAKE_CASE__								=   torch.tensor(
									    [[[-2.6_2_5_1, -1.4_2_9_8, -0.0_2_2_7], [-2.8_5_1_0, -1.6_3_8_7, 0.2_2_5_8], [-2.8_1_1_4, -1.1_8_3_2, -0.3_0_6_6]]]		)

									self.assertTrue(torch.allclose(output[:, :3, :3]				,					__A				,					atol=1E-4		)		)

'''simple docstring'''
import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation

lowercase__						       =      logging.get_logger(__name__)


lowercase__						       =      {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
    "tokenizer_config_file": "tokenizer_config.json",
}

lowercase__						       =      {
    "vocab_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"},
    "merges_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"},
    "tokenizer_config_file": {
        "facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"
    },
}

lowercase__						       =      {"facebook/blenderbot-3B": 1_2_8}




@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def    _UpperCamelCase						(       )  ->							Optional[Any]:


		'''simple docstring'''


		snake_case  :       Dict     			=						(
		    list(range(ord('''!'''			) ,       ord('''~'''			) + 1			)			) + list(range(ord('''¡'''			) ,       ord('''¬'''			) + 1			)			) + list(range(ord('''®'''			) ,       ord('''ÿ'''			) + 1			)			)
		)
		snake_case  :       int     			=						bs[:]
		snake_case  :       Any     			=						0
		for b in range(2**8			):
				if b not in bs:
						bs.append(SCREAMING_SNAKE_CASE__			)
						cs.append(2**8 + n			)
						n += 1
		snake_case  :       Tuple     			=						[chr(SCREAMING_SNAKE_CASE__			) for n in cs]
		return dict(zip(SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__			)			)




def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__			)  ->							Any:


		'''simple docstring'''


		snake_case  :       List[str]     			=						set()
		snake_case  :       str     			=						word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)			)
				snake_case  :       Dict     			=						char
		return pairs






class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""

		lowerCamelCase     							=						VOCAB_FILES_NAMES
		lowerCamelCase     							=						PRETRAINED_VOCAB_FILES_MAP
		lowerCamelCase     							=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowerCamelCase     							=						["""input_ids""", """attention_mask"""]



		def __init__(      self				:			Tuple  ,	UpperCamelCase__				:			Union[str, Any]  ,	UpperCamelCase__				:			List[Any]  ,	UpperCamelCase__				:			str="replace"  ,	UpperCamelCase__				:			int="<s>"  ,	UpperCamelCase__				:			Optional[int]="</s>"  ,	UpperCamelCase__				:			int="</s>"  ,	UpperCamelCase__				:			Optional[int]="<s>"  ,	UpperCamelCase__				:			Any="<unk>"  ,	UpperCamelCase__				:			List[Any]="<pad>"  ,	UpperCamelCase__				:			Optional[int]="<mask>"  ,	UpperCamelCase__				:			Optional[Any]=False  ,	**UpperCamelCase__				:			Dict  ,	)						->			str:






				"""simple docstring"""







				snake_case  :       str     			=						AddedToken(UpperCamelCase__  ,	lstrip=UpperCamelCase__  ,	rstrip=UpperCamelCase__      ) if isinstance(UpperCamelCase__  ,	UpperCamelCase__      ) else bos_token
				snake_case  :       Any     			=						AddedToken(UpperCamelCase__  ,	lstrip=UpperCamelCase__  ,	rstrip=UpperCamelCase__      ) if isinstance(UpperCamelCase__  ,	UpperCamelCase__      ) else eos_token
				snake_case  :       List[str]     			=						AddedToken(UpperCamelCase__  ,	lstrip=UpperCamelCase__  ,	rstrip=UpperCamelCase__      ) if isinstance(UpperCamelCase__  ,	UpperCamelCase__      ) else sep_token
				snake_case  :       Optional[Any]     			=						AddedToken(UpperCamelCase__  ,	lstrip=UpperCamelCase__  ,	rstrip=UpperCamelCase__      ) if isinstance(UpperCamelCase__  ,	UpperCamelCase__      ) else cls_token
				snake_case  :       int     			=						AddedToken(UpperCamelCase__  ,	lstrip=UpperCamelCase__  ,	rstrip=UpperCamelCase__      ) if isinstance(UpperCamelCase__  ,	UpperCamelCase__      ) else unk_token
				snake_case  :       str     			=						AddedToken(UpperCamelCase__  ,	lstrip=UpperCamelCase__  ,	rstrip=UpperCamelCase__      ) if isinstance(UpperCamelCase__  ,	UpperCamelCase__      ) else pad_token

				# Mask token behave like a normal word, i.e. include the space before it
				snake_case  :       str     			=						AddedToken(UpperCamelCase__  ,	lstrip=UpperCamelCase__  ,	rstrip=UpperCamelCase__      ) if isinstance(UpperCamelCase__  ,	UpperCamelCase__      ) else mask_token

				super().__init__(
				    errors=UpperCamelCase__  ,	bos_token=UpperCamelCase__  ,	eos_token=UpperCamelCase__  ,	unk_token=UpperCamelCase__  ,	sep_token=UpperCamelCase__  ,	cls_token=UpperCamelCase__  ,	pad_token=UpperCamelCase__  ,	mask_token=UpperCamelCase__  ,	add_prefix_space=UpperCamelCase__  ,	**UpperCamelCase__  ,	)

				with open(UpperCamelCase__  ,	encoding='''utf-8'''      ) as vocab_handle:
						snake_case  :       Union[str, Any]     			=						json.load(UpperCamelCase__      )
				snake_case  :       Dict     			=						{v: k for k, v in self.encoder.items()}
				snake_case  :       Optional[int]     			=						errors  # how to handle errors in decoding
				snake_case  :       List[Any]     			=						bytes_to_unicode()
				snake_case  :       Dict     			=						{v: k for k, v in self.byte_encoder.items()}
				with open(UpperCamelCase__  ,	encoding='''utf-8'''      ) as merges_handle:
						snake_case  :       Optional[Any]     			=						merges_handle.read().split('''\n'''      )[1:-1]
				snake_case  :       Optional[int]     			=						[tuple(merge.split()      ) for merge in bpe_merges]
				snake_case  :       Optional[int]     			=						dict(zip(UpperCamelCase__  ,	range(len(UpperCamelCase__      )      )      )      )
				snake_case  :       int     			=						{}
				snake_case  :       Any     			=						add_prefix_space

				# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
				snake_case  :       Any     			=						re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''      )



		@property
		# Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
		def       lowerCAmelCase				(      self				:			List[Any]      )						->			Any:






				"""simple docstring"""







				return len(self.encoder      )



		def       lowerCAmelCase				(      self				:			List[Any]      )						->			Optional[int]:






				"""simple docstring"""







				return dict(self.encoder  ,	**self.added_tokens_encoder      )



		def       lowerCAmelCase				(      self				:			Union[str, Any]  ,	UpperCamelCase__				:			Tuple      )						->			Dict:






				"""simple docstring"""







				if token in self.cache:
						return self.cache[token]
				snake_case  :       str     			=						tuple(UpperCamelCase__      )
				snake_case  :       Union[str, Any]     			=						get_pairs(UpperCamelCase__      )

				if not pairs:
						return token

				while True:
						snake_case  :       Tuple     			=						min(UpperCamelCase__  ,	key=lambda UpperCamelCase__      : self.bpe_ranks.get(UpperCamelCase__  ,	float('''inf'''      )      )      )
						if bigram not in self.bpe_ranks:
								break
						snake_case			,snake_case  :       List[str]     			=						bigram
						snake_case  :       int     			=						[]
						snake_case  :       int     			=						0
						while i < len(UpperCamelCase__      ):
								try:
										snake_case  :       Optional[int]     			=						word.index(UpperCamelCase__  ,	UpperCamelCase__      )
								except ValueError:
										new_word.extend(word[i:]      )
										break
								else:
										new_word.extend(word[i:j]      )
										snake_case  :       Optional[int]     			=						j

								if word[i] == first and i < len(UpperCamelCase__      ) - 1 and word[i + 1] == second:
										new_word.append(first + second      )
										i += 2
								else:
										new_word.append(word[i]      )
										i += 1
						snake_case  :       str     			=						tuple(UpperCamelCase__      )
						snake_case  :       Tuple     			=						new_word
						if len(UpperCamelCase__      ) == 1:
								break
						else:
								snake_case  :       List[str]     			=						get_pairs(UpperCamelCase__      )
				snake_case  :       str     			=						''' '''.join(UpperCamelCase__      )
				snake_case  :       int     			=						word
				return word



		def       lowerCAmelCase				(      self				:			Optional[int]  ,	UpperCamelCase__				:			List[Any]      )						->			List[str]:






				"""simple docstring"""







				snake_case  :       str     			=						[]
				for token in re.findall(self.pat  ,	UpperCamelCase__      ):
						snake_case  :       Optional[int]     			=						''''''.join(
						    self.byte_encoder[b] for b in token.encode('''utf-8'''      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
						bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase__      ).split(''' '''      )      )
				return bpe_tokens



		def       lowerCAmelCase				(      self				:			int  ,	UpperCamelCase__				:			Any      )						->			Any:






				"""simple docstring"""







				return self.encoder.get(UpperCamelCase__  ,	self.encoder.get(self.unk_token      )      )



		def       lowerCAmelCase				(      self				:			Any  ,	UpperCamelCase__				:			List[Any]      )						->			int:






				"""simple docstring"""







				return self.decoder.get(UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			Tuple  ,	UpperCamelCase__				:			int      )						->			List[str]:






				"""simple docstring"""







				snake_case  :       Optional[int]     			=						''''''.join(UpperCamelCase__      )
				snake_case  :       int     			=						bytearray([self.byte_decoder[c] for c in text]      ).decode('''utf-8'''  ,	errors=self.errors      )
				return text



		def       lowerCAmelCase				(      self				:			int  ,	UpperCamelCase__				:			str  ,	UpperCamelCase__				:			Optional[str] = None      )						->			Tuple[str]:






				"""simple docstring"""







				if not os.path.isdir(UpperCamelCase__      ):
						logger.error(f'Vocabulary path ({save_directory}) should be a directory'      )
						return
				snake_case  :       int     			=						os.path.join(
				    UpperCamelCase__  ,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )
				snake_case  :       List[Any]     			=						os.path.join(
				    UpperCamelCase__  ,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']      )

				with open(UpperCamelCase__  ,	'''w'''  ,	encoding='''utf-8'''      ) as f:
						f.write(json.dumps(self.encoder  ,	indent=2  ,	sort_keys=UpperCamelCase__  ,	ensure_ascii=UpperCamelCase__      ) + '''\n'''      )

				snake_case  :       str     			=						0
				with open(UpperCamelCase__  ,	'''w'''  ,	encoding='''utf-8'''      ) as writer:
						writer.write('''#version: 0.2\n'''      )
						for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,	key=lambda UpperCamelCase__      : kv[1]      ):
								if index != token_index:
										logger.warning(
										    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
										    ''' Please check that the tokenizer is not corrupted!'''      )
										snake_case  :       List[str]     			=						token_index
								writer.write(''' '''.join(UpperCamelCase__      ) + '''\n'''      )
								index += 1

				return vocab_file, merge_file



		def       lowerCAmelCase				(      self				:			List[Any]  ,	UpperCamelCase__				:			List[int]  ,	UpperCamelCase__				:			Optional[List[int]] = None  ,	UpperCamelCase__				:			bool = False      )						->			List[int]:






				"""simple docstring"""







				if already_has_special_tokens:
						return super().get_special_tokens_mask(
						    token_ids_a=UpperCamelCase__  ,	token_ids_a=UpperCamelCase__  ,	already_has_special_tokens=UpperCamelCase__      )

				if token_ids_a is None:
						return [1] + ([0] * len(UpperCamelCase__      )) + [1]
				return [1] + ([0] * len(UpperCamelCase__      )) + [1, 1] + ([0] * len(UpperCamelCase__      )) + [1]



		def       lowerCAmelCase				(      self				:			Optional[Any]  ,	UpperCamelCase__				:			List[int]  ,	UpperCamelCase__				:			Optional[List[int]] = None      )						->			List[int]:






				"""simple docstring"""







				snake_case  :       List[Any]     			=						[self.sep_token_id]
				snake_case  :       Tuple     			=						[self.cls_token_id]

				if token_ids_a is None:
						return len(cls + token_ids_a + sep      ) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]



		def       lowerCAmelCase				(      self				:			Optional[Any]  ,	UpperCamelCase__				:			Dict  ,	UpperCamelCase__				:			Any=False  ,	**UpperCamelCase__				:			List[Any]      )						->			int:






				"""simple docstring"""







				snake_case  :       Union[str, Any]     			=						kwargs.pop('''add_prefix_space'''  ,	self.add_prefix_space      )
				if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase__      ) > 0 and not text[0].isspace()):
						snake_case  :       Dict     			=						''' ''' + text
				return (text, kwargs)



		def       lowerCAmelCase				(      self				:			Optional[Any]  ,	UpperCamelCase__				:			List[int]  ,	UpperCamelCase__				:			Optional[List[int]] = None      )						->			str:






				"""simple docstring"""







				return token_ids_a + [self.eos_token_id]



		def       lowerCAmelCase				(      self				:			str  ,	UpperCamelCase__				:			"Conversation"      )						->			List[int]:






				"""simple docstring"""







				snake_case  :       Any     			=						[]
				for is_user, text in conversation.iter_texts():
						if is_user:
								# We need to space prefix as it's being done within blenderbot
								inputs.append(''' ''' + text      )
						else:
								# Generated responses should contain them already.
								inputs.append(UpperCamelCase__      )

				snake_case  :       List[str]     			=						'''  '''.join(UpperCamelCase__      )
				snake_case  :       Tuple     			=						self.encode(UpperCamelCase__      )
				if len(UpperCamelCase__      ) > self.model_max_length:
						snake_case  :       List[Any]     			=						input_ids[-self.model_max_length :]
						logger.warning(f'Trimmed input from conversation as it was longer than {self.model_max_length} tokens.'      )
				return input_ids


