



from ...configuration_utils import PretrainedConfig


a_   :List[Any]        =      {
    "google/tapas-base-finetuned-sqa": (
        "https://huggingface.co/google/tapas-base-finetuned-sqa/resolve/main/config.json"
    ),
    "google/tapas-base-finetuned-wtq": (
        "https://huggingface.co/google/tapas-base-finetuned-wtq/resolve/main/config.json"
    ),
    "google/tapas-base-finetuned-wikisql-supervised": (
        "https://huggingface.co/google/tapas-base-finetuned-wikisql-supervised/resolve/main/config.json"
    ),
    "google/tapas-base-finetuned-tabfact": (
        "https://huggingface.co/google/tapas-base-finetuned-tabfact/resolve/main/config.json"
    ),
}




class 					snake_case__						(       lowerCAmelCase_     ):






     """simple docstring"""


     _SCREAMING_SNAKE_CASE						=	"""tapas"""






     def __init__(       self  :  Optional[int], _snake_case  :  Union[str, Any]=3_0_5_2_2, _snake_case  :  Optional[Any]=7_6_8, _snake_case  :  List[Any]=1_2, _snake_case  :  Any=1_2, _snake_case  :  List[Any]=3_0_7_2, _snake_case  :  Dict="gelu", _snake_case  :  Dict=0.1, _snake_case  :  List[str]=0.1, _snake_case  :  Dict=1_0_2_4, _snake_case  :  Union[str, Any]=[3, 2_5_6, 2_5_6, 2, 2_5_6, 2_5_6, 1_0], _snake_case  :  Union[str, Any]=0.0_2, _snake_case  :  List[Any]=1e-12, _snake_case  :  List[Any]=0, _snake_case  :  int=1_0.0, _snake_case  :  Optional[int]=0, _snake_case  :  int=1.0, _snake_case  :  Tuple=None, _snake_case  :  Any=1.0, _snake_case  :  Tuple=False, _snake_case  :  Optional[Any]=None, _snake_case  :  Dict=1.0, _snake_case  :  List[str]=1.0, _snake_case  :  Any=False, _snake_case  :  str=False, _snake_case  :  List[str]="ratio", _snake_case  :  Dict=None, _snake_case  :  str=None, _snake_case  :  Dict=6_4, _snake_case  :  Dict=3_2, _snake_case  :  List[Any]=False, _snake_case  :  int=True, _snake_case  :  Tuple=False, _snake_case  :  Union[str, Any]=False, _snake_case  :  str=True, _snake_case  :  Any=False, _snake_case  :  Dict=None, _snake_case  :  int=None, **_snake_case  :  Optional[Any], )   ->Union[str, Any]:
            super().__init__(pad_token_id=_snake_case, **_snake_case					)

            # BERT hyperparameters (with updated max_position_embeddings and type_vocab_sizes)
            snake_case__							:	Optional[Any]   	=	vocab_size
            snake_case__							:	str   	=	hidden_size
            snake_case__							:	Union[str, Any]   	=	num_hidden_layers
            snake_case__							:	Optional[int]   	=	num_attention_heads
            snake_case__							:	Optional[int]   	=	hidden_act
            snake_case__							:	Optional[int]   	=	intermediate_size
            snake_case__							:	Optional[Any]   	=	hidden_dropout_prob
            snake_case__							:	Any   	=	attention_probs_dropout_prob
            snake_case__							:	Tuple   	=	max_position_embeddings
            snake_case__							:	str   	=	type_vocab_sizes
            snake_case__							:	Any   	=	initializer_range
            snake_case__							:	List[str]   	=	layer_norm_eps

            # Fine-tuning task hyperparameters
            snake_case__							:	Union[str, Any]   	=	positive_label_weight
            snake_case__							:	List[str]   	=	num_aggregation_labels
            snake_case__							:	int   	=	aggregation_loss_weight
            snake_case__							:	int   	=	use_answer_as_supervision
            snake_case__							:	int   	=	answer_loss_importance
            snake_case__							:	Optional[int]   	=	use_normalized_answer_loss
            snake_case__							:	int   	=	huber_loss_delta
            snake_case__							:	Union[str, Any]   	=	temperature
            snake_case__							:	List[Any]   	=	aggregation_temperature
            snake_case__							:	Optional[Any]   	=	use_gumbel_for_cells
            snake_case__							:	Any   	=	use_gumbel_for_aggregation
            snake_case__							:	str   	=	average_approximation_function
            snake_case__							:	int   	=	cell_selection_preference
            snake_case__							:	Any   	=	answer_loss_cutoff
            snake_case__							:	Optional[int]   	=	max_num_rows
            snake_case__							:	Any   	=	max_num_columns
            snake_case__							:	Optional[Any]   	=	average_logits_per_cell
            snake_case__							:	Dict   	=	select_one_column
            snake_case__							:	str   	=	allow_empty_column_selection
            snake_case__							:	Tuple   	=	init_cell_selection_weights_to_zero
            snake_case__							:	Any   	=	reset_position_index_per_cell
            snake_case__							:	Optional[Any]   	=	disable_per_token_loss

            # Aggregation hyperparameters
            snake_case__							:	str   	=	aggregation_labels
            snake_case__							:	Tuple   	=	no_aggregation_label_index

            if isinstance(self.aggregation_labels, _snake_case					):
                   snake_case__							:	List[Any]   	=	{int(_snake_case					): v for k, v in aggregation_labels.items()}











import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin




class 					snake_case__						(       lowerCAmelCase_				,     unittest.TestCase     ):






     """simple docstring"""


     _SCREAMING_SNAKE_CASE						=	TransfoXLTokenizer
     _SCREAMING_SNAKE_CASE						=	False
     _SCREAMING_SNAKE_CASE						=	False






     def    lowercase_   (       self  :  Optional[int]					)   ->Any:
            super().setUp()

            snake_case__							:	Tuple   	=	[
                '<unk>',
                '[CLS]',
                '[SEP]',
                'want',
                'unwanted',
                'wa',
                'un',
                'running',
                ',',
                'low',
                'l',
            ]
            snake_case__							:	Any   	=	os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['vocab_file']					)
            with open(self.vocab_file, 'w', encoding='utf-8'					) as vocab_writer:
                   vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)






     def    lowercase_   (       self  :  Union[str, Any], **_snake_case  :  List[Any]					)   ->Dict:
            snake_case__							:	str   	=	True
            return TransfoXLTokenizer.from_pretrained(self.tmpdirname, **_snake_case					)






     def    lowercase_   (       self  :  Optional[Any], _snake_case  :  str					)   ->Dict:
            snake_case__							:	List[Any]   	=	'<unk> UNwanted , running'
            snake_case__							:	List[Any]   	=	'<unk> unwanted, running'
            return input_text, output_text






     def    lowercase_   (       self  :  List[Any]					)   ->Tuple:
            snake_case__							:	Dict   	=	TransfoXLTokenizer(vocab_file=self.vocab_file, lower_case=_snake_case					)

            snake_case__							:	str   	=	tokenizer.tokenize('<unk> UNwanted , running'					)
            self.assertListEqual(_snake_case, ['<unk>', 'unwanted', ',', 'running']					)

            self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case					), [0, 4, 8, 7]					)






     def    lowercase_   (       self  :  List[str]					)   ->List[Any]:
            snake_case__							:	str   	=	TransfoXLTokenizer(lower_case=_snake_case					)

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '					), ['hello', '!', 'how', 'are', 'you', '?']					)






     def    lowercase_   (       self  :  Optional[int]					)   ->Optional[Any]:
            snake_case__							:	Optional[int]   	=	TransfoXLTokenizer(lower_case=_snake_case					)

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '					), ['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)






     def    lowercase_   (       self  :  Optional[int]					)   ->Union[str, Any]:
            snake_case__							:	List[Any]   	=	TransfoXLTokenizer(lower_case=_snake_case					)
            snake_case__							:	Dict   	=	'Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'
            snake_case__							:	List[Any]   	=	[
                'Hello',
                '(',
                'bracket',
                ')',
                'and',
                'side',
                '@-@',
                'scrolled',
                '[',
                'and',
                ']',
                'Henry',
                '\'s',
                '$',
                '5',
                '@,@',
                '000',
                'with',
                '3',
                '@.@',
                '34',
                'm',
                '.',
                'What',
                '\'s',
                'up',
                '!',
                '?',
            ]

            self.assertListEqual(tokenizer.tokenize(_snake_case					), _snake_case					)

            self.assertEqual(tokenizer.convert_tokens_to_string(_snake_case					), _snake_case					)






     def    lowercase_   (       self  :  Dict					)   ->Any:
            snake_case__							:	Dict   	=	self.get_tokenizer()
            snake_case__							:	Optional[Any]   	=	len(_snake_case					)

            tokenizer.add_tokens(['new1', 'new2']					)
            tokenizer.move_added_token('new1', 1					)

            # Check that moved token is not copied (duplicate)
            self.assertEqual(len(_snake_case					), original_len + 2					)
            # Check that token is moved to specified id
            self.assertEqual(tokenizer.encode('new1'					), [1]					)
            self.assertEqual(tokenizer.decode([1]					), 'new1'					)






