






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_fnet import FNetTokenizer
else:
							A_       :Optional[int]       =						None

A_       :int       =						logging.get_logger(__name__)
A_       :Any       =						{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

A_       :Dict       =						{
    '''vocab_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/spiece.model''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/tokenizer.json''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/tokenizer.json''',
    },
}

A_       :Optional[int]       =						{
    '''google/fnet-base''': 512,
    '''google/fnet-large''': 512,
}

A_       :Optional[int]       =						'''▁'''




class 				__A     (			a ):





				"""simple docstring"""







				UpperCamelCase__ :      Union[str, Any]			 =VOCAB_FILES_NAMES
				UpperCamelCase__ :      Union[str, Any]			 =PRETRAINED_VOCAB_FILES_MAP
				UpperCamelCase__ :      List[Any]			 =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCamelCase__ :      Tuple			 =["""input_ids""", """token_type_ids"""]
				UpperCamelCase__ :      Tuple			 =FNetTokenizer


				def __init__( self							,  lowerCamelCase__=None							,  lowerCamelCase__=None							,  lowerCamelCase__=False							,  lowerCamelCase__=True							,  lowerCamelCase__=True							,  lowerCamelCase__="<unk>"							,  lowerCamelCase__="[SEP]"							,  lowerCamelCase__="<pad>"							,  lowerCamelCase__="[CLS]"							,  lowerCamelCase__="[MASK]"							,  **lowerCamelCase__							,  ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[Any]       		=(
									    AddedToken(lowerCamelCase__							,  lstrip=lowerCamelCase__							,  rstrip=lowerCamelCase__							,  normalized=lowerCamelCase__    )
									    if isinstance(lowerCamelCase__							,  lowerCamelCase__    )
									    else mask_token
									)

									super().__init__(
									    lowerCamelCase__							,  tokenizer_file=lowerCamelCase__							,  do_lower_case=lowerCamelCase__							,  remove_space=lowerCamelCase__							,  keep_accents=lowerCamelCase__							,  unk_token=lowerCamelCase__							,  sep_token=lowerCamelCase__							,  pad_token=lowerCamelCase__							,  cls_token=lowerCamelCase__							,  mask_token=lowerCamelCase__							,  **lowerCamelCase__							,  )

									__UpperCamelCase      :					List[str]       		=do_lower_case
									__UpperCamelCase      :					Tuple       		=remove_space
									__UpperCamelCase      :					List[str]       		=keep_accents
									__UpperCamelCase      :					int       		=vocab_file
									__UpperCamelCase      :					Dict       		=False if not self.vocab_file else True


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__ = None    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[Any]       		=[self.sep_token_id]
									__UpperCamelCase      :					Any       		=[self.cls_token_id]
									if token_ids_a is None:
														return cls + token_ids_a + sep
									return cls + token_ids_a + sep + token_ids_a + sep


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__ = None    ):




									"""simple docstring"""

									__UpperCamelCase      :					Union[str, Any]       		=[self.sep_token_id]
									__UpperCamelCase      :					Tuple       		=[self.cls_token_id]

									if token_ids_a is None:
														return len(cls + token_ids_a + sep    ) * [0]
									return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__ = None    ):




									"""simple docstring"""

									if not os.path.isdir(lowerCamelCase__    ):
														logger.error(f'Vocabulary path ({save_directory}) should be a directory'    )
														return
									__UpperCamelCase      :					Optional[int]       		=os.path.join(
									    lowerCamelCase__							,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

									if os.path.abspath(self.vocab_file    ) != os.path.abspath(lowerCamelCase__    ):
														copyfile(self.vocab_file							,  lowerCamelCase__    )

									return (out_vocab_file,)








import unittest

import numpy as np

from transformers import AlbertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
					import jax.numpy as jnp

					from transformers.models.albert.modeling_flax_albert import (
					    FlaxAlbertForMaskedLM,
					    FlaxAlbertForMultipleChoice,
					    FlaxAlbertForPreTraining,
					    FlaxAlbertForQuestionAnswering,
					    FlaxAlbertForSequenceClassification,
					    FlaxAlbertForTokenClassification,
					    FlaxAlbertModel,
					)


class   A			(unittest.TestCase    ):
			'''simple docstring'''




			def __init__(			self   :    List[str]							,	__lowerCAmelCase   :    int							,	__lowerCAmelCase   :    List[str]=13							,	__lowerCAmelCase   :    int=7							,	__lowerCAmelCase   :    Tuple=True							,	__lowerCAmelCase   :    int=True							,	__lowerCAmelCase   :    Dict=True							,	__lowerCAmelCase   :    Union[str, Any]=True							,	__lowerCAmelCase   :    List[Any]=99							,	__lowerCAmelCase   :    Optional[Any]=32							,	__lowerCAmelCase   :    Optional[Any]=5							,	__lowerCAmelCase   :    Tuple=4							,	__lowerCAmelCase   :    Any=37							,	__lowerCAmelCase   :    str="gelu"							,	__lowerCAmelCase   :    Optional[Any]=0.1							,	__lowerCAmelCase   :    Union[str, Any]=0.1							,	__lowerCAmelCase   :    List[str]=5_12							,	__lowerCAmelCase   :    int=16							,	__lowerCAmelCase   :    Optional[int]=2							,	__lowerCAmelCase   :    List[Any]=0.0_2							,	__lowerCAmelCase   :    Tuple=4							,	)	->	Dict:







					"""simple docstring"""






					A__        =   parent
					A__        =   batch_size
					A__        =   seq_length
					A__        =   is_training
					A__        =   use_attention_mask
					A__        =   use_token_type_ids
					A__        =   use_labels
					A__        =   vocab_size
					A__        =   hidden_size
					A__        =   num_hidden_layers
					A__        =   num_attention_heads
					A__        =   intermediate_size
					A__        =   hidden_act
					A__        =   hidden_dropout_prob
					A__        =   attention_probs_dropout_prob
					A__        =   max_position_embeddings
					A__        =   type_vocab_size
					A__        =   type_sequence_label_size
					A__        =   initializer_range
					A__        =   num_choices






			def    a_       (			self   :    Any   )	->	str:







					"""simple docstring"""






					A__        =   ids_tensor([self.batch_size, self.seq_length]							,	self.vocab_size   )

					A__        =   None
					if self.use_attention_mask:
							A__        =   random_attention_mask([self.batch_size, self.seq_length]   )

					A__        =   None
					if self.use_token_type_ids:
							A__        =   ids_tensor([self.batch_size, self.seq_length]							,	self.type_vocab_size   )

					A__        =   AlbertConfig(
					    vocab_size=self.vocab_size							,	hidden_size=self.hidden_size							,	num_hidden_layers=self.num_hidden_layers							,	num_attention_heads=self.num_attention_heads							,	intermediate_size=self.intermediate_size							,	hidden_act=self.hidden_act							,	hidden_dropout_prob=self.hidden_dropout_prob							,	attention_probs_dropout_prob=self.attention_probs_dropout_prob							,	max_position_embeddings=self.max_position_embeddings							,	type_vocab_size=self.type_vocab_size							,	is_decoder=__lowerCAmelCase							,	initializer_range=self.initializer_range							,	)

					return config, input_ids, token_type_ids, attention_mask







			def    a_       (			self   :    Optional[int]   )	->	Optional[int]:







					"""simple docstring"""






					A__        =   self.prepare_config_and_inputs()
					A__      ,			A__      ,			A__      ,			A__        =   config_and_inputs
					A__        =   {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
					return config, inputs_dict


@require_flax
class   A			(SCREAMING_SNAKE_CASE   ,     unittest.TestCase    ):
			'''simple docstring'''




			__lowerCamelCase	:       str							=    (
			    (
			        FlaxAlbertModel,
			        FlaxAlbertForPreTraining,
			        FlaxAlbertForMaskedLM,
			        FlaxAlbertForMultipleChoice,
			        FlaxAlbertForQuestionAnswering,
			        FlaxAlbertForSequenceClassification,
			        FlaxAlbertForTokenClassification,
			        FlaxAlbertForQuestionAnswering,
			    )
			    if is_flax_available()
			    else ()
			)






			def    a_       (			self   :    str   )	->	Optional[int]:







					"""simple docstring"""






					A__        =   FlaxAlbertModelTester(self   )







			@slow
			def    a_       (			self   :    int   )	->	Tuple:







					"""simple docstring"""






					for model_class_name in self.all_model_classes:
							A__        =   model_class_name.from_pretrained("""albert-base-v2"""   )
							A__        =   model(np.ones((1, 1)   )   )
							self.assertIsNotNone(__lowerCAmelCase   )


@require_flax
class   A			(unittest.TestCase    ):
			'''simple docstring'''




			@slow
			def    a_       (			self   :    Dict   )	->	List[Any]:







					"""simple docstring"""






					A__        =   FlaxAlbertModel.from_pretrained("""albert-base-v2"""   )
					A__        =   np.array([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]   )
					A__        =   np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]   )
					A__        =   model(__lowerCAmelCase							,	attention_mask=__lowerCAmelCase   )[0]
					A__        =   (1, 11, 7_68)
					self.assertEqual(output.shape							,	__lowerCAmelCase   )
					A__        =   np.array(
					    [[[-0.6_5_1_3, 1.5_0_3_5, -0.2_7_6_6], [-0.6_5_1_5, 1.5_0_4_6, -0.2_7_8_0], [-0.6_5_1_2, 1.5_0_4_9, -0.2_7_8_4]]]   )

					self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]							,	__lowerCAmelCase							,	atol=1e-4   )   )



