


from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import numpy as np
				import tensorflow as tf

				from transformers import (
				    TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    FlaubertConfig,
				    TFFlaubertForMultipleChoice,
				    TFFlaubertForQuestionAnsweringSimple,
				    TFFlaubertForSequenceClassification,
				    TFFlaubertForTokenClassification,
				    TFFlaubertModel,
				    TFFlaubertWithLMHeadModel,
				)






class    _UpperCamelCase     :

				"""simple docstring"""







				def __init__(    self					, a__					, )						->					Any:
					A             =					parent
					A             =					13
					A             =					7
					A             =					True
					A             =					True
					A             =					True
					A             =					True
					A             =					True
					A             =					False
					A             =					False
					A             =					False
					A             =					2
					A             =					99
					A             =					0
					A             =					32
					A             =					2
					A             =					4
					A             =					0.1
					A             =					0.1
					A             =					512
					A             =					16
					A             =					2
					A             =					0.02
					A             =					3
					A             =					4
					A             =					"""last"""
					A             =					True
					A             =					None
					A             =					0

				def   _UpperCAmelCase       (    self )						->					List[Any]:
					A             =					ids_tensor([self.batch_size, self.seq_length]					, self.vocab_size )
					A             =					random_attention_mask([self.batch_size, self.seq_length]					, dtype=tf.floataa )

					A             =					None
					if self.use_input_lengths:
						A             =					(
						    ids_tensor([self.batch_size]					, vocab_size=2 ) + self.seq_length - 2
						)  # small variation of seq_length

					A             =					None
					if self.use_token_type_ids:
						A             =					ids_tensor([self.batch_size, self.seq_length]					, self.n_langs )

					A             =					None
					A             =					None
					A             =					None
					if self.use_labels:
						A             =					ids_tensor([self.batch_size]					, self.type_sequence_label_size )
						A             =					ids_tensor([self.batch_size, self.seq_length]					, self.num_labels )
						A             =					ids_tensor([self.batch_size]					, 2					, dtype=tf.floataa )
						A             =					ids_tensor([self.batch_size]					, self.num_choices )

					A             =					FlaubertConfig(
					    vocab_size=self.vocab_size					, n_special=self.n_special					, emb_dim=self.hidden_size					, n_layers=self.num_hidden_layers					, n_heads=self.num_attention_heads					, dropout=self.hidden_dropout_prob					, attention_dropout=self.attention_probs_dropout_prob					, gelu_activation=self.gelu_activation					, sinusoidal_embeddings=self.sinusoidal_embeddings					, asm=self.asm					, causal=self.causal					, n_langs=self.n_langs					, max_position_embeddings=self.max_position_embeddings					, initializer_range=self.initializer_range					, summary_type=self.summary_type					, use_proj=self.use_proj					, bos_token_id=self.bos_token_id					, )

					return (
					    config,
					    input_ids,
					    token_type_ids,
					    input_lengths,
					    sequence_labels,
					    token_labels,
					    is_impossible_labels,
					    choice_labels,
					    input_mask,
					)

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, )						->					Union[str, Any]:
					A             =					TFFlaubertModel(config=a__ )
					A             =					{"""input_ids""": input_ids, """lengths""": input_lengths, """langs""": token_type_ids}
					A             =					model(a__ )

					A             =					[input_ids, input_mask]
					A             =					model(a__ )
					self.parent.assertEqual(result.last_hidden_state.shape					, (self.batch_size, self.seq_length, self.hidden_size) )

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, )						->					Union[str, Any]:
					A             =					TFFlaubertWithLMHeadModel(a__ )

					A             =					{"""input_ids""": input_ids, """lengths""": input_lengths, """langs""": token_type_ids}
					A             =					model(a__ )

					self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.seq_length, self.vocab_size) )

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, )						->					Any:
					A             =					TFFlaubertForQuestionAnsweringSimple(a__ )

					A             =					{"""input_ids""": input_ids, """lengths""": input_lengths}

					A             =					model(a__ )

					self.parent.assertEqual(result.start_logits.shape					, (self.batch_size, self.seq_length) )
					self.parent.assertEqual(result.end_logits.shape					, (self.batch_size, self.seq_length) )

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, )						->					Any:
					A             =					TFFlaubertForSequenceClassification(a__ )

					A             =					{"""input_ids""": input_ids, """lengths""": input_lengths}

					A             =					model(a__ )

					self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.type_sequence_label_size) )

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, )						->					Tuple:
					A             =					self.num_labels
					A             =					TFFlaubertForTokenClassification(config=a__ )
					A             =					{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
					A             =					model(a__ )
					self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.seq_length, self.num_labels) )

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, a__					, )						->					Any:
					A             =					self.num_choices
					A             =					TFFlaubertForMultipleChoice(config=a__ )
					A             =					tf.tile(tf.expand_dims(a__					, 1 )					, (1, self.num_choices, 1) )
					A             =					tf.tile(tf.expand_dims(a__					, 1 )					, (1, self.num_choices, 1) )
					A             =					tf.tile(tf.expand_dims(a__					, 1 )					, (1, self.num_choices, 1) )
					A             =					{
					    """input_ids""": multiple_choice_inputs_ids,
					    """attention_mask""": multiple_choice_input_mask,
					    """token_type_ids""": multiple_choice_token_type_ids,
					}
					A             =					model(a__ )
					self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.num_choices) )

				def   _UpperCAmelCase       (    self )						->					Any:
					A             =					self.prepare_config_and_inputs()
					(
					    (
					    A
					)       ,      (
					    A
					)       ,      (
					    A
					)       ,      (
					    A
					)       ,      (
					    A
					)       ,      (
					    A
					)       ,      (
					    A
					)       ,      (
					    A
					)       ,      (
					    A
					)       ,      
					)             =					config_and_inputs
					A             =					{
					    """input_ids""": input_ids,
					    """token_type_ids""": token_type_ids,
					    """langs""": token_type_ids,
					    """lengths""": input_lengths,
					}
					return config, inputs_dict






@require_tf
class    _UpperCamelCase     (       __snake_case     ,		__snake_case     ,		unittest.TestCase					):

				"""simple docstring"""







				lowerCAmelCase					=			(
				    (
				        TFFlaubertModel,
				        TFFlaubertWithLMHeadModel,
				        TFFlaubertForSequenceClassification,
				        TFFlaubertForQuestionAnsweringSimple,
				        TFFlaubertForTokenClassification,
				        TFFlaubertForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				lowerCAmelCase					=			(
				    (TFFlaubertWithLMHeadModel,) if is_tf_available() else ()
				)  # TODO (PVP): Check other models whether language generation is also applicable
				lowerCAmelCase					=			(
				    {
				        'feature-extraction': TFFlaubertModel,
				        'fill-mask': TFFlaubertWithLMHeadModel,
				        'question-answering': TFFlaubertForQuestionAnsweringSimple,
				        'text-classification': TFFlaubertForSequenceClassification,
				        'token-classification': TFFlaubertForTokenClassification,
				        'zero-shot': TFFlaubertForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)
				lowerCAmelCase					=			False
				lowerCAmelCase					=			False

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__ )						->					Union[str, Any]:
					if (
					    pipeline_test_casse_name == "QAPipelineTests"
					    and tokenizer_name is not None
					    and not tokenizer_name.endswith("""Fast""" )
					):
						# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
						# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
						# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
						return True

					return False

				def   _UpperCAmelCase       (    self )						->					Tuple:
					A             =					TFFlaubertModelTester(self )
					A             =					ConfigTester(self					, config_class=a__					, emb_dim=37 )

				def   _UpperCAmelCase       (    self )						->					List[str]:
					self.config_tester.run_common_tests()

				def   _UpperCAmelCase       (    self )						->					str:
					A             =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_flaubert_model(*a__ )

				def   _UpperCAmelCase       (    self )						->					str:
					A             =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_flaubert_lm_head(*a__ )

				def   _UpperCAmelCase       (    self )						->					Optional[int]:
					A             =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_flaubert_qa(*a__ )

				def   _UpperCAmelCase       (    self )						->					Any:
					A             =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_flaubert_sequence_classif(*a__ )

				def   _UpperCAmelCase       (    self )						->					List[str]:
					A             =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_flaubert_for_token_classification(*a__ )

				def   _UpperCAmelCase       (    self )						->					Any:
					A             =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_flaubert_for_multiple_choice(*a__ )

				@slow
				def   _UpperCAmelCase       (    self )						->					int:
					for model_name in TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						A             =					TFFlaubertModel.from_pretrained(a__ )
						self.assertIsNotNone(a__ )






@require_tf
@require_sentencepiece
@require_tokenizers
class    _UpperCamelCase     (       unittest.TestCase					):

				"""simple docstring"""







				@slow
				def   _UpperCAmelCase       (    self )						->					Any:
					A             =					TFFlaubertModel.from_pretrained("""jplu/tf-flaubert-small-cased""" )

					A             =					tf.convert_to_tensor(
					    [[0, 158, 735, 2592, 1424, 6727, 82, 1]]					, dtype=tf.intaa					, )  # "J'aime flaubert !"

					A             =					model(a__ )[0]
					A             =					tf.TensorShape((1, 8, 512) )
					self.assertEqual(output.shape					, a__ )
					# compare the actual values for a slice.
					A             =					tf.convert_to_tensor(
					    [
					        [
					            [-1.8_76_87_73, -1.56_65_55, 0.27_07_24_18],
					            [-1.6_92_00_38, -0.5_87_35_05, 1.9_32_95_99],
					            [-2.9_56_39_85, -1.6_99_38_35, 1.7_97_20_52],
					        ]
					    ]					, dtype=tf.floataa					, )

					self.assertTrue(np.allclose(output[:, :3, :3].numpy()					, expected_slice.numpy()					, atol=1e-4 ) )







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
				from .tokenization_albert import AlbertTokenizer
else:
				_lowercase							:   List[str]						=						None

_lowercase							:   List[str]						=						logging.get_logger(__name__)
_lowercase							:   List[Any]						=						{"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

_lowercase							:   str						=						{
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    },
    "tokenizer_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json",
    },
}

_lowercase							:   List[str]						=						{
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

_lowercase							:   Dict						=						"▁"






class    _UpperCamelCase     (       __snake_case					):

				"""simple docstring"""







				lowerCAmelCase					=			VOCAB_FILES_NAMES
				lowerCAmelCase					=			PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase					=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase					=			AlbertTokenizer

				def __init__(    self					, a__=None					, a__=None					, a__=True					, a__=True					, a__=False					, a__="[CLS]"					, a__="[SEP]"					, a__="<unk>"					, a__="[SEP]"					, a__="<pad>"					, a__="[CLS]"					, a__="[MASK]"					, **a__					, )						->					Dict:
					# Mask token behave like a normal word, i.e. include the space before it and
					# is included in the raw text, there should be a match in a non-normalized sentence.
					A             =					(
					    AddedToken(a__					, lstrip=a__					, rstrip=a__					, normalized=a__ )
					    if isinstance(a__					, a__ )
					    else mask_token
					)

					super().__init__(
					    a__					, tokenizer_file=a__					, do_lower_case=a__					, remove_space=a__					, keep_accents=a__					, bos_token=a__					, eos_token=a__					, unk_token=a__					, sep_token=a__					, pad_token=a__					, cls_token=a__					, mask_token=a__					, **a__					, )

					A             =					do_lower_case
					A             =					remove_space
					A             =					keep_accents
					A             =					vocab_file
					A             =					False if not self.vocab_file else True

				def   _UpperCAmelCase       (    self					, a__					, a__ = None )						->					List[int]:
					A             =					[self.sep_token_id]
					A             =					[self.cls_token_id]
					if token_ids_a is None:
						return cls + token_ids_a + sep
					return cls + token_ids_a + sep + token_ids_a + sep

				def   _UpperCAmelCase       (    self					, a__					, a__ = None )						->					List[int]:
					A             =					[self.sep_token_id]
					A             =					[self.cls_token_id]

					if token_ids_a is None:
						return len(cls + token_ids_a + sep ) * [0]
					return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]

				def   _UpperCAmelCase       (    self					, a__					, a__ = None )						->					Tuple[str]:
					if not self.can_save_slow_tokenizer:
						raise ValueError(
						    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
						    """tokenizer.""" )

					if not os.path.isdir(a__ ):
						logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
						return
					A             =					os.path.join(
					    a__					, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

					if os.path.abspath(self.vocab_file ) != os.path.abspath(a__ ):
						copyfile(self.vocab_file					, a__ )

					return (out_vocab_file,)



