




from collections import OrderedDict

from ...utils import logging
from .auto_factory import _BaseAutoModelClass, _LazyAutoMapping, auto_class_update
from .configuration_auto import CONFIG_MAPPING_NAMES


UpperCamelCase__    			=			logging.get_logger(__name__)


UpperCamelCase__    			=			OrderedDict(
    [
        # Base model mapping
        ("""albert""", """FlaxAlbertModel"""),
        ("""bart""", """FlaxBartModel"""),
        ("""beit""", """FlaxBeitModel"""),
        ("""bert""", """FlaxBertModel"""),
        ("""big_bird""", """FlaxBigBirdModel"""),
        ("""blenderbot""", """FlaxBlenderbotModel"""),
        ("""blenderbot-small""", """FlaxBlenderbotSmallModel"""),
        ("""clip""", """FlaxCLIPModel"""),
        ("""distilbert""", """FlaxDistilBertModel"""),
        ("""electra""", """FlaxElectraModel"""),
        ("""gpt-sw3""", """FlaxGPT2Model"""),
        ("""gpt2""", """FlaxGPT2Model"""),
        ("""gpt_neo""", """FlaxGPTNeoModel"""),
        ("""gptj""", """FlaxGPTJModel"""),
        ("""longt5""", """FlaxLongT5Model"""),
        ("""marian""", """FlaxMarianModel"""),
        ("""mbart""", """FlaxMBartModel"""),
        ("""mt5""", """FlaxMT5Model"""),
        ("""opt""", """FlaxOPTModel"""),
        ("""pegasus""", """FlaxPegasusModel"""),
        ("""regnet""", """FlaxRegNetModel"""),
        ("""resnet""", """FlaxResNetModel"""),
        ("""roberta""", """FlaxRobertaModel"""),
        ("""roberta-prelayernorm""", """FlaxRobertaPreLayerNormModel"""),
        ("""roformer""", """FlaxRoFormerModel"""),
        ("""t5""", """FlaxT5Model"""),
        ("""vision-text-dual-encoder""", """FlaxVisionTextDualEncoderModel"""),
        ("""vit""", """FlaxViTModel"""),
        ("""wav2vec2""", """FlaxWav2Vec2Model"""),
        ("""whisper""", """FlaxWhisperModel"""),
        ("""xglm""", """FlaxXGLMModel"""),
        ("""xlm-roberta""", """FlaxXLMRobertaModel"""),
    ]
)

UpperCamelCase__    			=			OrderedDict(
    [
        # Model for pre-training mapping
        ("""albert""", """FlaxAlbertForPreTraining"""),
        ("""bart""", """FlaxBartForConditionalGeneration"""),
        ("""bert""", """FlaxBertForPreTraining"""),
        ("""big_bird""", """FlaxBigBirdForPreTraining"""),
        ("""electra""", """FlaxElectraForPreTraining"""),
        ("""longt5""", """FlaxLongT5ForConditionalGeneration"""),
        ("""mbart""", """FlaxMBartForConditionalGeneration"""),
        ("""mt5""", """FlaxMT5ForConditionalGeneration"""),
        ("""roberta""", """FlaxRobertaForMaskedLM"""),
        ("""roberta-prelayernorm""", """FlaxRobertaPreLayerNormForMaskedLM"""),
        ("""roformer""", """FlaxRoFormerForMaskedLM"""),
        ("""t5""", """FlaxT5ForConditionalGeneration"""),
        ("""wav2vec2""", """FlaxWav2Vec2ForPreTraining"""),
        ("""whisper""", """FlaxWhisperForConditionalGeneration"""),
        ("""xlm-roberta""", """FlaxXLMRobertaForMaskedLM"""),
    ]
)

UpperCamelCase__    			=			OrderedDict(
    [
        # Model for Masked LM mapping
        ("""albert""", """FlaxAlbertForMaskedLM"""),
        ("""bart""", """FlaxBartForConditionalGeneration"""),
        ("""bert""", """FlaxBertForMaskedLM"""),
        ("""big_bird""", """FlaxBigBirdForMaskedLM"""),
        ("""distilbert""", """FlaxDistilBertForMaskedLM"""),
        ("""electra""", """FlaxElectraForMaskedLM"""),
        ("""mbart""", """FlaxMBartForConditionalGeneration"""),
        ("""roberta""", """FlaxRobertaForMaskedLM"""),
        ("""roberta-prelayernorm""", """FlaxRobertaPreLayerNormForMaskedLM"""),
        ("""roformer""", """FlaxRoFormerForMaskedLM"""),
        ("""xlm-roberta""", """FlaxXLMRobertaForMaskedLM"""),
    ]
)

UpperCamelCase__    			=			OrderedDict(
    [
        # Model for Seq2Seq Causal LM mapping
        ("""bart""", """FlaxBartForConditionalGeneration"""),
        ("""blenderbot""", """FlaxBlenderbotForConditionalGeneration"""),
        ("""blenderbot-small""", """FlaxBlenderbotSmallForConditionalGeneration"""),
        ("""encoder-decoder""", """FlaxEncoderDecoderModel"""),
        ("""longt5""", """FlaxLongT5ForConditionalGeneration"""),
        ("""marian""", """FlaxMarianMTModel"""),
        ("""mbart""", """FlaxMBartForConditionalGeneration"""),
        ("""mt5""", """FlaxMT5ForConditionalGeneration"""),
        ("""pegasus""", """FlaxPegasusForConditionalGeneration"""),
        ("""t5""", """FlaxT5ForConditionalGeneration"""),
    ]
)

UpperCamelCase__    			=			OrderedDict(
    [
        # Model for Image-classsification
        ("""beit""", """FlaxBeitForImageClassification"""),
        ("""regnet""", """FlaxRegNetForImageClassification"""),
        ("""resnet""", """FlaxResNetForImageClassification"""),
        ("""vit""", """FlaxViTForImageClassification"""),
    ]
)

UpperCamelCase__    			=			OrderedDict(
    [
        ("""vision-encoder-decoder""", """FlaxVisionEncoderDecoderModel"""),
    ]
)

UpperCamelCase__    			=			OrderedDict(
    [
        # Model for Causal LM mapping
        ("""bart""", """FlaxBartForCausalLM"""),
        ("""bert""", """FlaxBertForCausalLM"""),
        ("""big_bird""", """FlaxBigBirdForCausalLM"""),
        ("""electra""", """FlaxElectraForCausalLM"""),
        ("""gpt-sw3""", """FlaxGPT2LMHeadModel"""),
        ("""gpt2""", """FlaxGPT2LMHeadModel"""),
        ("""gpt_neo""", """FlaxGPTNeoForCausalLM"""),
        ("""gptj""", """FlaxGPTJForCausalLM"""),
        ("""opt""", """FlaxOPTForCausalLM"""),
        ("""roberta""", """FlaxRobertaForCausalLM"""),
        ("""roberta-prelayernorm""", """FlaxRobertaPreLayerNormForCausalLM"""),
        ("""xglm""", """FlaxXGLMForCausalLM"""),
        ("""xlm-roberta""", """FlaxXLMRobertaForCausalLM"""),
    ]
)

UpperCamelCase__    			=			OrderedDict(
    [
        # Model for Sequence Classification mapping
        ("""albert""", """FlaxAlbertForSequenceClassification"""),
        ("""bart""", """FlaxBartForSequenceClassification"""),
        ("""bert""", """FlaxBertForSequenceClassification"""),
        ("""big_bird""", """FlaxBigBirdForSequenceClassification"""),
        ("""distilbert""", """FlaxDistilBertForSequenceClassification"""),
        ("""electra""", """FlaxElectraForSequenceClassification"""),
        ("""mbart""", """FlaxMBartForSequenceClassification"""),
        ("""roberta""", """FlaxRobertaForSequenceClassification"""),
        ("""roberta-prelayernorm""", """FlaxRobertaPreLayerNormForSequenceClassification"""),
        ("""roformer""", """FlaxRoFormerForSequenceClassification"""),
        ("""xlm-roberta""", """FlaxXLMRobertaForSequenceClassification"""),
    ]
)

UpperCamelCase__    			=			OrderedDict(
    [
        # Model for Question Answering mapping
        ("""albert""", """FlaxAlbertForQuestionAnswering"""),
        ("""bart""", """FlaxBartForQuestionAnswering"""),
        ("""bert""", """FlaxBertForQuestionAnswering"""),
        ("""big_bird""", """FlaxBigBirdForQuestionAnswering"""),
        ("""distilbert""", """FlaxDistilBertForQuestionAnswering"""),
        ("""electra""", """FlaxElectraForQuestionAnswering"""),
        ("""mbart""", """FlaxMBartForQuestionAnswering"""),
        ("""roberta""", """FlaxRobertaForQuestionAnswering"""),
        ("""roberta-prelayernorm""", """FlaxRobertaPreLayerNormForQuestionAnswering"""),
        ("""roformer""", """FlaxRoFormerForQuestionAnswering"""),
        ("""xlm-roberta""", """FlaxXLMRobertaForQuestionAnswering"""),
    ]
)

UpperCamelCase__    			=			OrderedDict(
    [
        # Model for Token Classification mapping
        ("""albert""", """FlaxAlbertForTokenClassification"""),
        ("""bert""", """FlaxBertForTokenClassification"""),
        ("""big_bird""", """FlaxBigBirdForTokenClassification"""),
        ("""distilbert""", """FlaxDistilBertForTokenClassification"""),
        ("""electra""", """FlaxElectraForTokenClassification"""),
        ("""roberta""", """FlaxRobertaForTokenClassification"""),
        ("""roberta-prelayernorm""", """FlaxRobertaPreLayerNormForTokenClassification"""),
        ("""roformer""", """FlaxRoFormerForTokenClassification"""),
        ("""xlm-roberta""", """FlaxXLMRobertaForTokenClassification"""),
    ]
)

UpperCamelCase__    			=			OrderedDict(
    [
        # Model for Multiple Choice mapping
        ("""albert""", """FlaxAlbertForMultipleChoice"""),
        ("""bert""", """FlaxBertForMultipleChoice"""),
        ("""big_bird""", """FlaxBigBirdForMultipleChoice"""),
        ("""distilbert""", """FlaxDistilBertForMultipleChoice"""),
        ("""electra""", """FlaxElectraForMultipleChoice"""),
        ("""roberta""", """FlaxRobertaForMultipleChoice"""),
        ("""roberta-prelayernorm""", """FlaxRobertaPreLayerNormForMultipleChoice"""),
        ("""roformer""", """FlaxRoFormerForMultipleChoice"""),
        ("""xlm-roberta""", """FlaxXLMRobertaForMultipleChoice"""),
    ]
)

UpperCamelCase__    			=			OrderedDict(
    [
        ("""bert""", """FlaxBertForNextSentencePrediction"""),
    ]
)

UpperCamelCase__    			=			OrderedDict(
    [
        ("""speech-encoder-decoder""", """FlaxSpeechEncoderDecoderModel"""),
        ("""whisper""", """FlaxWhisperForConditionalGeneration"""),
    ]
)

UpperCamelCase__    			=			OrderedDict(
    [
        ("""whisper""", """FlaxWhisperForAudioClassification"""),
    ]
)

UpperCamelCase__    			=			_LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_MAPPING_NAMES)
UpperCamelCase__    			=			_LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_PRETRAINING_MAPPING_NAMES)
UpperCamelCase__    			=			_LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_MASKED_LM_MAPPING_NAMES)
UpperCamelCase__    			=			_LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES
)
UpperCamelCase__    			=			_LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES
)
UpperCamelCase__    			=			_LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES)
UpperCamelCase__    			=			_LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
UpperCamelCase__    			=			_LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES
)
UpperCamelCase__    			=			_LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES
)
UpperCamelCase__    			=			_LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES
)
UpperCamelCase__    			=			_LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES
)
UpperCamelCase__    			=			_LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES
)
UpperCamelCase__    			=			_LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES
)
UpperCamelCase__    			=			_LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES
)
class 			a__    (					_BaseAutoModelClass      ):
						_a							:    Tuple					=    FLAX_MODEL_MAPPING






UpperCamelCase__    			=			auto_class_update(FlaxAutoModel)
class 			a__    (					_BaseAutoModelClass      ):
						_a							:    Any					=    FLAX_MODEL_FOR_PRETRAINING_MAPPING






UpperCamelCase__    			=			auto_class_update(FlaxAutoModelForPreTraining, head_doc="""pretraining""")
class 			a__    (					_BaseAutoModelClass      ):
						_a							:    List[Any]					=    FLAX_MODEL_FOR_CAUSAL_LM_MAPPING






UpperCamelCase__    			=			auto_class_update(FlaxAutoModelForCausalLM, head_doc="""causal language modeling""")
class 			a__    (					_BaseAutoModelClass      ):
						_a							:    Optional[int]					=    FLAX_MODEL_FOR_MASKED_LM_MAPPING






UpperCamelCase__    			=			auto_class_update(FlaxAutoModelForMaskedLM, head_doc="""masked language modeling""")
class 			a__    (					_BaseAutoModelClass      ):
						_a							:    Dict					=    FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING






UpperCamelCase__    			=			auto_class_update(
    FlaxAutoModelForSeqaSeqLM, head_doc="""sequence-to-sequence language modeling""", checkpoint_for_example="""t5-base"""
)
class 			a__    (					_BaseAutoModelClass      ):
						_a							:    Any					=    FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING






UpperCamelCase__    			=			auto_class_update(
    FlaxAutoModelForSequenceClassification, head_doc="""sequence classification"""
)
class 			a__    (					_BaseAutoModelClass      ):
						_a							:    int					=    FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING






UpperCamelCase__    			=			auto_class_update(FlaxAutoModelForQuestionAnswering, head_doc="""question answering""")
class 			a__    (					_BaseAutoModelClass      ):
						_a							:    Tuple					=    FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING






UpperCamelCase__    			=			auto_class_update(
    FlaxAutoModelForTokenClassification, head_doc="""token classification"""
)
class 			a__    (					_BaseAutoModelClass      ):
						_a							:    int					=    FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING






UpperCamelCase__    			=			auto_class_update(FlaxAutoModelForMultipleChoice, head_doc="""multiple choice""")
class 			a__    (					_BaseAutoModelClass      ):
						_a							:    List[Any]					=    FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING






UpperCamelCase__    			=			auto_class_update(
    FlaxAutoModelForNextSentencePrediction, head_doc="""next sentence prediction"""
)
class 			a__    (					_BaseAutoModelClass      ):
						_a							:    Tuple					=    FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING






UpperCamelCase__    			=			auto_class_update(
    FlaxAutoModelForImageClassification, head_doc="""image classification"""
)
class 			a__    (					_BaseAutoModelClass      ):
						_a							:    int					=    FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING






UpperCamelCase__    			=			auto_class_update(FlaxAutoModelForVisionaSeq, head_doc="""vision-to-text modeling""")
class 			a__    (					_BaseAutoModelClass      ):
						_a							:    int					=    FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING






UpperCamelCase__    			=			auto_class_update(
    FlaxAutoModelForSpeechSeqaSeq, head_doc="""sequence-to-sequence speech-to-text modeling"""
)










import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__    			=			get_tests_dir("""fixtures/test_sentencepiece.model""")
UpperCamelCase__    			=			get_tests_dir("""fixtures/test_sentencepiece_bpe.model""")

UpperCamelCase__    			=			"""pt""" if is_torch_available() else """tf"""
@require_sentencepiece
@require_tokenizers
class 			a__    (					snake_case__ ,				unittest.TestCase      ):
						_a							:    int					=    CamembertTokenizer
						_a							:    Dict					=    CamembertTokenizerFast
						_a							:    Tuple					=    True
						_a							:    List[Any]					=    True






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											super().setUp()

											# We have a SentencePiece fixture for testing
											__lowerCAmelCase				   =       CamembertTokenizer(_A			)
											tokenizer.save_pretrained(self.tmpdirname			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "<pad>"
											__lowerCAmelCase				   =       1

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A			)		,			_A			)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A			)		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       list(self.get_tokenizer().get_vocab().keys()			)

											self.assertEqual(vocab_keys[0]		,			"<s>NOTUSED"			)
											self.assertEqual(vocab_keys[1]		,			"<pad>"			)
											self.assertEqual(vocab_keys[-1]		,			"<mask>"			)
											self.assertEqual(len(_A			)		,			1_0_0_4			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											self.assertEqual(self.get_tokenizer().vocab_size		,			1_0_0_5			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       CamembertTokenizer(_A			)
											tokenizer.save_pretrained(self.tmpdirname			)
											__lowerCAmelCase				   =       CamembertTokenizerFast.from_pretrained(self.tmpdirname			)

											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."

											__lowerCAmelCase				   =       tokenizer.encode(_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)

											# <unk> tokens are not the same for `rust` than for `slow`.
											# Because spm gives back raw token instead of `unk` in EncodeAsPieces
											# tokens = tokenizer.tokenize(sequence)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(_A			)
											__lowerCAmelCase				   =       rust_tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											if not self.test_rust_tokenizer:
																return

											__lowerCAmelCase				   =       self.get_tokenizer()
											__lowerCAmelCase				   =       self.get_rust_tokenizer()

											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."

											__lowerCAmelCase				   =       tokenizer.tokenize(_A			)
											__lowerCAmelCase				   =       rust_tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       self.get_rust_tokenizer()
											__lowerCAmelCase				   =       tokenizer.encode(_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A			)
											self.assertListEqual(_A		,			_A			)






						@slow
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       {"input_ids": [[5, 5_4, 7_1_9_6, 2_9_7, 3_0, 2_3, 7_7_6, 1_8, 1_1, 3_2_1_5, 3_7_0_5, 8_2_5_2, 2_2, 3_1_6_4, 1_1_8_1, 2_1_1_6, 2_9, 1_6, 8_1_3, 2_5, 7_9_1, 3_3_1_4, 2_0, 3_4_4_6, 3_8, 2_7_5_7_5, 1_2_0, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 4_6_8, 1_7, 1_1, 9_0_8_8, 2_0, 1_5_1_7, 8, 2_2_8_0_4, 1_8_8_1_8, 1_0, 3_8, 6_2_9, 6_0_7, 6_0_7, 1_4_2, 1_9, 7_1_9_6, 8_6_7, 5_6, 1_0_3_2_6, 2_4, 2_2_6_7, 2_0, 4_1_6, 5_0_7_2, 1_5_6_1_2, 2_3_3, 7_3_4, 7, 2_3_9_9, 2_7, 1_6, 3_0_1_5, 1_6_4_9, 7, 2_4, 2_0, 4_3_3_8, 2_3_9_9, 2_7, 1_3, 3_4_0_0, 1_4, 1_3, 6_1_8_9, 8, 9_3_0, 9, 6]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
											# fmt: on

											# camembert is a french model. So we also use french texts.
											__lowerCAmelCase				   =       [
											    "Le transformeur est un modèle d'apprentissage profond introduit en 2017, "
											    "utilisé principalement dans le domaine du traitement automatique des langues (TAL).",
											    "À l'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus "
											    "pour gérer des données séquentielles, telles que le langage naturel, pour des tâches "
											    "telles que la traduction et la synthèse de texte.",
											]

											self.tokenizer_integration_test_util(
											    expected_encoding=_A		,			model_name="camembert-base"		,			revision="3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf"		,			sequences=_A		,			)




