





from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import numpy as np
					import tensorflow as tf

					from transformers import (
					    TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
					    FlaubertConfig,
					    TFFlaubertForMultipleChoice,
					    TFFlaubertForQuestionAnsweringSimple,
					    TFFlaubertForSequenceClassification,
					    TFFlaubertForTokenClassification,
					    TFFlaubertModel,
					    TFFlaubertWithLMHeadModel,
					)


class   A			:
			'''simple docstring'''




			def __init__(			self   :    Tuple							,	__lowerCAmelCase   :    int							,	)	->	Union[str, Any]:







					"""simple docstring"""






					A__        =   parent
					A__        =   13
					A__        =   7
					A__        =   True
					A__        =   True
					A__        =   True
					A__        =   True
					A__        =   True
					A__        =   False
					A__        =   False
					A__        =   False
					A__        =   2
					A__        =   99
					A__        =   0
					A__        =   32
					A__        =   2
					A__        =   4
					A__        =   0.1
					A__        =   0.1
					A__        =   5_12
					A__        =   16
					A__        =   2
					A__        =   0.0_2
					A__        =   3
					A__        =   4
					A__        =   """last"""
					A__        =   True
					A__        =   None
					A__        =   0






			def    a_       (			self   :    Any   )	->	Optional[Any]:







					"""simple docstring"""






					A__        =   ids_tensor([self.batch_size, self.seq_length]							,	self.vocab_size   )
					A__        =   random_attention_mask([self.batch_size, self.seq_length]							,	dtype=tf.floataa   )

					A__        =   None
					if self.use_input_lengths:
							A__        =   (
							    ids_tensor([self.batch_size]							,	vocab_size=2   ) + self.seq_length - 2
							)  # small variation of seq_length

					A__        =   None
					if self.use_token_type_ids:
							A__        =   ids_tensor([self.batch_size, self.seq_length]							,	self.n_langs   )

					A__        =   None
					A__        =   None
					A__        =   None
					if self.use_labels:
							A__        =   ids_tensor([self.batch_size]							,	self.type_sequence_label_size   )
							A__        =   ids_tensor([self.batch_size, self.seq_length]							,	self.num_labels   )
							A__        =   ids_tensor([self.batch_size]							,	2							,	dtype=tf.floataa   )
							A__        =   ids_tensor([self.batch_size]							,	self.num_choices   )

					A__        =   FlaubertConfig(
					    vocab_size=self.vocab_size							,	n_special=self.n_special							,	emb_dim=self.hidden_size							,	n_layers=self.num_hidden_layers							,	n_heads=self.num_attention_heads							,	dropout=self.hidden_dropout_prob							,	attention_dropout=self.attention_probs_dropout_prob							,	gelu_activation=self.gelu_activation							,	sinusoidal_embeddings=self.sinusoidal_embeddings							,	asm=self.asm							,	causal=self.causal							,	n_langs=self.n_langs							,	max_position_embeddings=self.max_position_embeddings							,	initializer_range=self.initializer_range							,	summary_type=self.summary_type							,	use_proj=self.use_proj							,	bos_token_id=self.bos_token_id							,	)

					return (
					    config,
					    input_ids,
					    token_type_ids,
					    input_lengths,
					    sequence_labels,
					    token_labels,
					    is_impossible_labels,
					    choice_labels,
					    input_mask,
					)






			def    a_       (			self   :    Union[str, Any]							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    List[str]							,	__lowerCAmelCase   :    Optional[Any]							,	__lowerCAmelCase   :    Tuple							,	__lowerCAmelCase   :    Optional[Any]							,	__lowerCAmelCase   :    int							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    Any							,	__lowerCAmelCase   :    List[Any]							,	)	->	Optional[Any]:







					"""simple docstring"""






					A__        =   TFFlaubertModel(config=__lowerCAmelCase   )
					A__        =   {"""input_ids""": input_ids, """lengths""": input_lengths, """langs""": token_type_ids}
					A__        =   model(__lowerCAmelCase   )

					A__        =   [input_ids, input_mask]
					A__        =   model(__lowerCAmelCase   )
					self.parent.assertEqual(result.last_hidden_state.shape							,	(self.batch_size, self.seq_length, self.hidden_size)   )






			def    a_       (			self   :    int							,	__lowerCAmelCase   :    Tuple							,	__lowerCAmelCase   :    int							,	__lowerCAmelCase   :    List[Any]							,	__lowerCAmelCase   :    List[str]							,	__lowerCAmelCase   :    Tuple							,	__lowerCAmelCase   :    int							,	__lowerCAmelCase   :    List[str]							,	__lowerCAmelCase   :    str							,	__lowerCAmelCase   :    Tuple							,	)	->	Tuple:







					"""simple docstring"""






					A__        =   TFFlaubertWithLMHeadModel(__lowerCAmelCase   )

					A__        =   {"""input_ids""": input_ids, """lengths""": input_lengths, """langs""": token_type_ids}
					A__        =   model(__lowerCAmelCase   )

					self.parent.assertEqual(result.logits.shape							,	(self.batch_size, self.seq_length, self.vocab_size)   )






			def    a_       (			self   :    List[Any]							,	__lowerCAmelCase   :    List[Any]							,	__lowerCAmelCase   :    int							,	__lowerCAmelCase   :    List[Any]							,	__lowerCAmelCase   :    List[str]							,	__lowerCAmelCase   :    List[str]							,	__lowerCAmelCase   :    Any							,	__lowerCAmelCase   :    List[Any]							,	__lowerCAmelCase   :    List[Any]							,	__lowerCAmelCase   :    Union[str, Any]							,	)	->	str:







					"""simple docstring"""






					A__        =   TFFlaubertForQuestionAnsweringSimple(__lowerCAmelCase   )

					A__        =   {"""input_ids""": input_ids, """lengths""": input_lengths}

					A__        =   model(__lowerCAmelCase   )

					self.parent.assertEqual(result.start_logits.shape							,	(self.batch_size, self.seq_length)   )
					self.parent.assertEqual(result.end_logits.shape							,	(self.batch_size, self.seq_length)   )






			def    a_       (			self   :    Any							,	__lowerCAmelCase   :    Optional[Any]							,	__lowerCAmelCase   :    List[Any]							,	__lowerCAmelCase   :    List[str]							,	__lowerCAmelCase   :    str							,	__lowerCAmelCase   :    Dict							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    Optional[Any]							,	__lowerCAmelCase   :    Dict							,	__lowerCAmelCase   :    Any							,	)	->	Dict:







					"""simple docstring"""






					A__        =   TFFlaubertForSequenceClassification(__lowerCAmelCase   )

					A__        =   {"""input_ids""": input_ids, """lengths""": input_lengths}

					A__        =   model(__lowerCAmelCase   )

					self.parent.assertEqual(result.logits.shape							,	(self.batch_size, self.type_sequence_label_size)   )






			def    a_       (			self   :    Optional[int]							,	__lowerCAmelCase   :    Union[str, Any]							,	__lowerCAmelCase   :    int							,	__lowerCAmelCase   :    Dict							,	__lowerCAmelCase   :    Optional[Any]							,	__lowerCAmelCase   :    List[str]							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    Union[str, Any]							,	__lowerCAmelCase   :    int							,	__lowerCAmelCase   :    Optional[Any]							,	)	->	Any:







					"""simple docstring"""






					A__        =   self.num_labels
					A__        =   TFFlaubertForTokenClassification(config=__lowerCAmelCase   )
					A__        =   {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
					A__        =   model(__lowerCAmelCase   )
					self.parent.assertEqual(result.logits.shape							,	(self.batch_size, self.seq_length, self.num_labels)   )






			def    a_       (			self   :    List[Any]							,	__lowerCAmelCase   :    str							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    int							,	__lowerCAmelCase   :    str							,	__lowerCAmelCase   :    str							,	__lowerCAmelCase   :    Union[str, Any]							,	__lowerCAmelCase   :    Optional[int]							,	)	->	Union[str, Any]:







					"""simple docstring"""






					A__        =   self.num_choices
					A__        =   TFFlaubertForMultipleChoice(config=__lowerCAmelCase   )
					A__        =   tf.tile(tf.expand_dims(__lowerCAmelCase							,	1   )							,	(1, self.num_choices, 1)   )
					A__        =   tf.tile(tf.expand_dims(__lowerCAmelCase							,	1   )							,	(1, self.num_choices, 1)   )
					A__        =   tf.tile(tf.expand_dims(__lowerCAmelCase							,	1   )							,	(1, self.num_choices, 1)   )
					A__        =   {
					    """input_ids""": multiple_choice_inputs_ids,
					    """attention_mask""": multiple_choice_input_mask,
					    """token_type_ids""": multiple_choice_token_type_ids,
					}
					A__        =   model(__lowerCAmelCase   )
					self.parent.assertEqual(result.logits.shape							,	(self.batch_size, self.num_choices)   )







			def    a_       (			self   :    List[Any]   )	->	Dict:







					"""simple docstring"""






					A__        =   self.prepare_config_and_inputs()
					(
					    (
					    A__
					)      ,			(
					    A__
					)      ,			(
					    A__
					)      ,			(
					    A__
					)      ,			(
					    A__
					)      ,			(
					    A__
					)      ,			(
					    A__
					)      ,			(
					    A__
					)      ,			(
					    A__
					)      ,			
					)        =   config_and_inputs
					A__        =   {
					    """input_ids""": input_ids,
					    """token_type_ids""": token_type_ids,
					    """langs""": token_type_ids,
					    """lengths""": input_lengths,
					}
					return config, inputs_dict


@require_tf
class   A			(SCREAMING_SNAKE_CASE   ,     SCREAMING_SNAKE_CASE   ,     unittest.TestCase    ):
			'''simple docstring'''




			__lowerCamelCase	:       str							=    (
			    (
			        TFFlaubertModel,
			        TFFlaubertWithLMHeadModel,
			        TFFlaubertForSequenceClassification,
			        TFFlaubertForQuestionAnsweringSimple,
			        TFFlaubertForTokenClassification,
			        TFFlaubertForMultipleChoice,
			    )
			    if is_tf_available()
			    else ()
			)
			__lowerCamelCase	:       str							=    (
			    (TFFlaubertWithLMHeadModel,) if is_tf_available() else ()
			)  # TODO (PVP): Check other models whether language generation is also applicable
			__lowerCamelCase	:       List[Any]							=    (
			    {
			        '''feature-extraction''': TFFlaubertModel,
			        '''fill-mask''': TFFlaubertWithLMHeadModel,
			        '''question-answering''': TFFlaubertForQuestionAnsweringSimple,
			        '''text-classification''': TFFlaubertForSequenceClassification,
			        '''token-classification''': TFFlaubertForTokenClassification,
			        '''zero-shot''': TFFlaubertForSequenceClassification,
			    }
			    if is_tf_available()
			    else {}
			)
			__lowerCamelCase	:       Optional[int]							=    False
			__lowerCamelCase	:       Optional[Any]							=    False






			def    a_       (			self   :    str							,	__lowerCAmelCase   :    Any							,	__lowerCAmelCase   :    Dict							,	__lowerCAmelCase   :    Optional[Any]							,	__lowerCAmelCase   :    Union[str, Any]							,	__lowerCAmelCase   :    Optional[int]   )	->	str:







					"""simple docstring"""






					if (
					    pipeline_test_casse_name == "QAPipelineTests"
					    and tokenizer_name is not None
					    and not tokenizer_name.endswith("""Fast"""   )
					):
							# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
							# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
							# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
							return True

					return False






			def    a_       (			self   :    List[str]   )	->	Union[str, Any]:







					"""simple docstring"""






					A__        =   TFFlaubertModelTester(self   )
					A__        =   ConfigTester(self							,	config_class=__lowerCAmelCase							,	emb_dim=37   )






			def    a_       (			self   :    Optional[int]   )	->	Union[str, Any]:







					"""simple docstring"""






					self.config_tester.run_common_tests()






			def    a_       (			self   :    Optional[Any]   )	->	Dict:







					"""simple docstring"""






					A__        =   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_flaubert_model(*__lowerCAmelCase   )






			def    a_       (			self   :    Optional[Any]   )	->	Optional[int]:







					"""simple docstring"""






					A__        =   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_flaubert_lm_head(*__lowerCAmelCase   )






			def    a_       (			self   :    Any   )	->	List[Any]:







					"""simple docstring"""






					A__        =   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_flaubert_qa(*__lowerCAmelCase   )






			def    a_       (			self   :    str   )	->	Dict:







					"""simple docstring"""






					A__        =   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_flaubert_sequence_classif(*__lowerCAmelCase   )






			def    a_       (			self   :    Tuple   )	->	List[Any]:







					"""simple docstring"""






					A__        =   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_flaubert_for_token_classification(*__lowerCAmelCase   )






			def    a_       (			self   :    int   )	->	Union[str, Any]:







					"""simple docstring"""






					A__        =   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_flaubert_for_multiple_choice(*__lowerCAmelCase   )







			@slow
			def    a_       (			self   :    int   )	->	List[Any]:







					"""simple docstring"""






					for model_name in TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							A__        =   TFFlaubertModel.from_pretrained(__lowerCAmelCase   )
							self.assertIsNotNone(__lowerCAmelCase   )


@require_tf
@require_sentencepiece
@require_tokenizers
class   A			(unittest.TestCase    ):
			'''simple docstring'''




			@slow
			def    a_       (			self   :    Any   )	->	str:







					"""simple docstring"""






					A__        =   TFFlaubertModel.from_pretrained("""jplu/tf-flaubert-small-cased"""   )

					A__        =   tf.convert_to_tensor(
					    [[0, 1_58, 7_35, 25_92, 14_24, 67_27, 82, 1]]							,	dtype=tf.intaa							,	)  # "J'aime flaubert !"

					A__        =   model(__lowerCAmelCase   )[0]
					A__        =   tf.TensorShape((1, 8, 5_12)   )
					self.assertEqual(output.shape							,	__lowerCAmelCase   )
					# compare the actual values for a slice.
					A__        =   tf.convert_to_tensor(
					    [
					        [
					            [-1.8_7_6_8_7_7_3, -1.5_6_6_5_5_5, 0.2_7_0_7_2_4_1_8],
					            [-1.6_9_2_0_0_3_8, -0.5_8_7_3_5_0_5, 1.9_3_2_9_5_9_9],
					            [-2.9_5_6_3_9_8_5, -1.6_9_9_3_8_3_5, 1.7_9_7_2_0_5_2],
					        ]
					    ]							,	dtype=tf.floataa							,	)

					self.assertTrue(np.allclose(output[:, :3, :3].numpy()							,	expected_slice.numpy()							,	atol=1e-4   )   )










import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_albert import AlbertTokenizer
else:
					A    :					Tuple =      None

A    :					Optional[Any] =      logging.get_logger(__name__)
A    :					Tuple =      {'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

A    :					List[str] =      {
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json''',
    },
}

A    :					List[str] =      {
    '''albert-base-v1''': 5_1_2,
    '''albert-large-v1''': 5_1_2,
    '''albert-xlarge-v1''': 5_1_2,
    '''albert-xxlarge-v1''': 5_1_2,
    '''albert-base-v2''': 5_1_2,
    '''albert-large-v2''': 5_1_2,
    '''albert-xlarge-v2''': 5_1_2,
    '''albert-xxlarge-v2''': 5_1_2,
}

A    :					Optional[int] =      '''▁'''


class   A			(SCREAMING_SNAKE_CASE    ):
			'''simple docstring'''




			__lowerCamelCase	:       str							=    VOCAB_FILES_NAMES
			__lowerCamelCase	:       List[Any]							=    PRETRAINED_VOCAB_FILES_MAP
			__lowerCamelCase	:       Dict							=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__lowerCamelCase	:       List[str]							=    AlbertTokenizer






			def __init__(			self   :    Tuple							,	__lowerCAmelCase   :    Tuple=None							,	__lowerCAmelCase   :    List[str]=None							,	__lowerCAmelCase   :    Optional[Any]=True							,	__lowerCAmelCase   :    List[Any]=True							,	__lowerCAmelCase   :    str=False							,	__lowerCAmelCase   :    Union[str, Any]="[CLS]"							,	__lowerCAmelCase   :    int="[SEP]"							,	__lowerCAmelCase   :    Dict="<unk>"							,	__lowerCAmelCase   :    Dict="[SEP]"							,	__lowerCAmelCase   :    Union[str, Any]="<pad>"							,	__lowerCAmelCase   :    str="[CLS]"							,	__lowerCAmelCase   :    int="[MASK]"							,	**__lowerCAmelCase   :    Optional[Any]							,	)	->	Optional[Any]:







					"""simple docstring"""






					A__        =   (
					    AddedToken(__lowerCAmelCase							,	lstrip=__lowerCAmelCase							,	rstrip=__lowerCAmelCase							,	normalized=__lowerCAmelCase   )
					    if isinstance(__lowerCAmelCase							,	__lowerCAmelCase   )
					    else mask_token
					)

					super().__init__(
					    __lowerCAmelCase							,	tokenizer_file=__lowerCAmelCase							,	do_lower_case=__lowerCAmelCase							,	remove_space=__lowerCAmelCase							,	keep_accents=__lowerCAmelCase							,	bos_token=__lowerCAmelCase							,	eos_token=__lowerCAmelCase							,	unk_token=__lowerCAmelCase							,	sep_token=__lowerCAmelCase							,	pad_token=__lowerCAmelCase							,	cls_token=__lowerCAmelCase							,	mask_token=__lowerCAmelCase							,	**__lowerCAmelCase							,	)

					A__        =   do_lower_case
					A__        =   remove_space
					A__        =   keep_accents
					A__        =   vocab_file
					A__        =   False if not self.vocab_file else True






			def    a_       (			self   :    List[Any]							,	__lowerCAmelCase   :    List[int]							,	__lowerCAmelCase   :    Optional[List[int]] = None   )	->	List[int]:







					"""simple docstring"""






					A__        =   [self.sep_token_id]
					A__        =   [self.cls_token_id]
					if token_ids_a is None:
							return cls + token_ids_a + sep
					return cls + token_ids_a + sep + token_ids_a + sep






			def    a_       (			self   :    Union[str, Any]							,	__lowerCAmelCase   :    List[int]							,	__lowerCAmelCase   :    Optional[List[int]] = None   )	->	List[int]:







					"""simple docstring"""






					A__        =   [self.sep_token_id]
					A__        =   [self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep   ) * [0]
					return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]







			def    a_       (			self   :    Tuple							,	__lowerCAmelCase   :    str							,	__lowerCAmelCase   :    Optional[str] = None   )	->	Tuple[str]:







					"""simple docstring"""






					if not self.can_save_slow_tokenizer:
							raise ValueError(
							    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
							    """tokenizer."""   )

					if not os.path.isdir(__lowerCAmelCase   ):
							logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
							return
					A__        =   os.path.join(
					    __lowerCAmelCase							,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )

					if os.path.abspath(self.vocab_file   ) != os.path.abspath(__lowerCAmelCase   ):
							copyfile(self.vocab_file							,	__lowerCAmelCase   )

					return (out_vocab_file,)



