
'''simple docstring'''




import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 					A							(				__snake_case       ,     unittest.TestCase		):
     __magic_name__          =						GPTaTokenizer
     __magic_name__          =						GPTaTokenizerFast
     __magic_name__          =						True
     __magic_name__          =						{'''add_prefix_space''': True}
     __magic_name__          =						False





     def 							__lowerCAmelCase				(				self      )					->							Any:






         """simple docstring"""






         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         A			:  Dict   =		[
             '''l''',
             '''o''',
             '''w''',
             '''e''',
             '''r''',
             '''s''',
             '''t''',
             '''i''',
             '''d''',
             '''n''',
             '''\u0120''',
             '''\u0120l''',
             '''\u0120n''',
             '''\u0120lo''',
             '''\u0120low''',
             '''er''',
             '''\u0120lowest''',
             '''\u0120newer''',
             '''\u0120wider''',
             '''<unk>''',
             '''<|endoftext|>''',
         ]
         A			:  Optional[Any]   =		dict(zip(SCREAMING_SNAKE_CASE		,     range(len(SCREAMING_SNAKE_CASE      )      )      )      )
         A			:  str   =		['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
         A			:  List[str]   =		{'''unk_token''': '''<unk>'''}

         A			:  Optional[Any]   =		os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES['''vocab_file''']      )
         A			:  str   =		os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES['''merges_file''']      )
         with open(self.vocab_file		,     '''w'''		,     encoding='''utf-8'''      ) as fp:
             fp.write(json.dumps(SCREAMING_SNAKE_CASE      ) + '''\n'''      )
         with open(self.merges_file		,     '''w'''		,     encoding='''utf-8'''      ) as fp:
             fp.write('''\n'''.join(SCREAMING_SNAKE_CASE      )      )





     def 							__lowerCAmelCase				(				self		,     **SCREAMING_SNAKE_CASE      )					->							Union[str, Any]:






         """simple docstring"""






         kwargs.update(self.special_tokens_map      )
         return GPTaTokenizer.from_pretrained(self.tmpdirname		,     **SCREAMING_SNAKE_CASE      )





     def 							__lowerCAmelCase				(				self		,     **SCREAMING_SNAKE_CASE      )					->							List[Any]:






         """simple docstring"""






         kwargs.update(self.special_tokens_map      )
         return GPTaTokenizerFast.from_pretrained(self.tmpdirname		,     **SCREAMING_SNAKE_CASE      )





     def 							__lowerCAmelCase				(				self		,     SCREAMING_SNAKE_CASE      )					->							List[str]:






         """simple docstring"""






         A			:  Dict   =		'''lower newer'''
         A			:  Tuple   =		'''lower newer'''
         return input_text, output_text





     def 							__lowerCAmelCase				(				self      )					->							Any:






         """simple docstring"""






         A			:  str   =		GPTaTokenizer(self.vocab_file		,     self.merges_file		,     **self.special_tokens_map      )
         A			:  Optional[Any]   =		'''lower newer'''
         A			:  Tuple   =		['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
         A			:  Union[str, Any]   =		tokenizer.tokenize(SCREAMING_SNAKE_CASE		,     add_prefix_space=SCREAMING_SNAKE_CASE      )
         self.assertListEqual(SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )

         A			:  List[str]   =		tokens + [tokenizer.unk_token]
         A			:  str   =		[14, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE      )		,     SCREAMING_SNAKE_CASE      )





     def 							__lowerCAmelCase				(				self      )					->							Optional[Any]:






         """simple docstring"""






         if not self.test_rust_tokenizer:
             return

         A			:  Optional[int]   =		self.get_tokenizer()
         A			:  Any   =		self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE      )

         A			:  List[str]   =		'''lower newer'''

         # Testing tokenization
         A			:  Union[str, Any]   =		tokenizer.tokenize(SCREAMING_SNAKE_CASE		,     add_prefix_space=SCREAMING_SNAKE_CASE      )
         A			:  Union[str, Any]   =		rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE      )
         self.assertListEqual(SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )

         # Testing conversion to ids without special tokens
         A			:  Optional[int]   =		tokenizer.encode(SCREAMING_SNAKE_CASE		,     add_special_tokens=SCREAMING_SNAKE_CASE		,     add_prefix_space=SCREAMING_SNAKE_CASE      )
         A			:  Any   =		rust_tokenizer.encode(SCREAMING_SNAKE_CASE		,     add_special_tokens=SCREAMING_SNAKE_CASE      )
         self.assertListEqual(SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )

         # Testing conversion to ids with special tokens
         A			:  Tuple   =		self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE      )
         A			:  Optional[int]   =		tokenizer.encode(SCREAMING_SNAKE_CASE		,     add_prefix_space=SCREAMING_SNAKE_CASE      )
         A			:  Any   =		rust_tokenizer.encode(SCREAMING_SNAKE_CASE      )
         self.assertListEqual(SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )

         # Testing the unknown token
         A			:  Optional[Any]   =		tokens + [rust_tokenizer.unk_token]
         A			:  Optional[Any]   =		[14, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE      )		,     SCREAMING_SNAKE_CASE      )





     def 							__lowerCAmelCase				(				self		,     *SCREAMING_SNAKE_CASE		,     **SCREAMING_SNAKE_CASE      )					->							Tuple:






         """simple docstring"""






         pass





     def 							__lowerCAmelCase				(				self		,     SCREAMING_SNAKE_CASE=15      )					->							List[Any]:






         """simple docstring"""






         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
                 A			:  Tuple   =		self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE		,     **SCREAMING_SNAKE_CASE      )

                 # Simple input
                 A			:  List[str]   =		'''This is a simple input'''
                 A			:  Any   =		['''This is a simple input 1''', '''This is a simple input 2''']
                 A			:  Optional[Any]   =		('''This is a simple input''', '''This is a pair''')
                 A			:  Any   =		[
                     ('''This is a simple input 1''', '''This is a simple input 2'''),
                     ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                 ]

                 # Simple input tests
                 self.assertRaises(SCREAMING_SNAKE_CASE		,     tokenizer_r.encode		,     SCREAMING_SNAKE_CASE		,     max_length=SCREAMING_SNAKE_CASE		,     padding='''max_length'''      )

                 # Simple input
                 self.assertRaises(SCREAMING_SNAKE_CASE		,     tokenizer_r.encode_plus		,     SCREAMING_SNAKE_CASE		,     max_length=SCREAMING_SNAKE_CASE		,     padding='''max_length'''      )

                 # Simple input
                 self.assertRaises(
                     SCREAMING_SNAKE_CASE		,     tokenizer_r.batch_encode_plus		,     SCREAMING_SNAKE_CASE		,     max_length=SCREAMING_SNAKE_CASE		,     padding='''max_length'''		,     )

                 # Pair input
                 self.assertRaises(SCREAMING_SNAKE_CASE		,     tokenizer_r.encode		,     SCREAMING_SNAKE_CASE		,     max_length=SCREAMING_SNAKE_CASE		,     padding='''max_length'''      )

                 # Pair input
                 self.assertRaises(SCREAMING_SNAKE_CASE		,     tokenizer_r.encode_plus		,     SCREAMING_SNAKE_CASE		,     max_length=SCREAMING_SNAKE_CASE		,     padding='''max_length'''      )

                 # Pair input
                 self.assertRaises(
                     SCREAMING_SNAKE_CASE		,     tokenizer_r.batch_encode_plus		,     SCREAMING_SNAKE_CASE		,     max_length=SCREAMING_SNAKE_CASE		,     padding='''max_length'''		,     )





     def 							__lowerCAmelCase				(				self      )					->							int:






         """simple docstring"""






         A			:  List[Any]   =		GPTaTokenizer.from_pretrained(self.tmpdirname		,     pad_token='''<pad>'''      )

         # Simple input
         A			:  Any   =		'''This is a simple input'''
         A			:  Tuple   =		['''This is a simple input looooooooong''', '''This is a simple input''']
         A			:  int   =		('''This is a simple input''', '''This is a pair''')
         A			:  int   =		[
             ('''This is a simple input loooooong''', '''This is a simple input'''),
             ('''This is a simple pair loooooong''', '''This is a simple pair'''),
         ]

         A			:  Union[str, Any]   =		tokenizer.pad_token_id

         A			:  List[str]   =		tokenizer(SCREAMING_SNAKE_CASE		,     padding='''max_length'''		,     max_length=30		,     return_tensors='''np'''      )
         A			:  Tuple   =		tokenizer(SCREAMING_SNAKE_CASE		,     padding=SCREAMING_SNAKE_CASE		,     truncate=SCREAMING_SNAKE_CASE		,     return_tensors='''np'''      )
         A			:  str   =		tokenizer(*SCREAMING_SNAKE_CASE		,     padding='''max_length'''		,     max_length=60		,     return_tensors='''np'''      )
         A			:  Dict   =		tokenizer(SCREAMING_SNAKE_CASE		,     padding=SCREAMING_SNAKE_CASE		,     truncate=SCREAMING_SNAKE_CASE		,     return_tensors='''np'''      )

         # s
         # test single string max_length padding
         self.assertEqual(out_s['''input_ids'''].shape[-1]		,     30      )
         self.assertTrue(pad_token_id in out_s['''input_ids''']      )
         self.assertTrue(0 in out_s['''attention_mask''']      )

         # s2
         # test automatic padding
         self.assertEqual(out_sa['''input_ids'''].shape[-1]		,     33      )
         # long slice doesn't have padding
         self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]      )
         self.assertFalse(0 in out_sa['''attention_mask'''][0]      )
         # short slice does have padding
         self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]      )
         self.assertTrue(0 in out_sa['''attention_mask'''][1]      )

         # p
         # test single pair max_length padding
         self.assertEqual(out_p['''input_ids'''].shape[-1]		,     60      )
         self.assertTrue(pad_token_id in out_p['''input_ids''']      )
         self.assertTrue(0 in out_p['''attention_mask''']      )

         # p2
         # test automatic padding pair
         self.assertEqual(out_pa['''input_ids'''].shape[-1]		,     52      )
         # long slice pair doesn't have padding
         self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]      )
         self.assertFalse(0 in out_pa['''attention_mask'''][0]      )
         # short slice pair does have padding
         self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]      )
         self.assertTrue(0 in out_pa['''attention_mask'''][1]      )





     def 							__lowerCAmelCase				(				self      )					->							str:






         """simple docstring"""






         A			:  Tuple   =		'''$$$'''
         A			:  Optional[Any]   =		GPTaTokenizer.from_pretrained(self.tmpdirname		,     bos_token=SCREAMING_SNAKE_CASE		,     add_bos_token=SCREAMING_SNAKE_CASE      )

         A			:  Optional[int]   =		'''This is a simple input'''
         A			:  List[str]   =		['''This is a simple input 1''', '''This is a simple input 2''']

         A			:  str   =		tokenizer.bos_token_id

         A			:  Optional[Any]   =		tokenizer(SCREAMING_SNAKE_CASE      )
         A			:  Tuple   =		tokenizer(SCREAMING_SNAKE_CASE      )

         self.assertEqual(out_s.input_ids[0]		,     SCREAMING_SNAKE_CASE      )
         self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids      )      )

         A			:  Union[str, Any]   =		tokenizer.decode(out_s.input_ids      )
         A			:  str   =		tokenizer.batch_decode(out_sa.input_ids      )

         self.assertEqual(decode_s.split()[0]		,     SCREAMING_SNAKE_CASE      )
         self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa      )      )





     def 							__lowerCAmelCase				(				self      )					->							List[str]:






         """simple docstring"""






         pass





     def 							__lowerCAmelCase				(				self      )					->							Optional[Any]:






         """simple docstring"""






         A			:  Optional[int]   =		[self.get_tokenizer(do_lower_case=SCREAMING_SNAKE_CASE		,     add_bos_token=SCREAMING_SNAKE_CASE      )]
         for tokenizer in tokenizers:
             with self.subTest(F'{tokenizer.__class__.__name__}'      ):
                 A			:  List[str]   =		'''Encode this.'''
                 A			:  int   =		'''This one too please.'''
                 A			:  Optional[Any]   =		tokenizer.encode(SCREAMING_SNAKE_CASE		,     add_special_tokens=SCREAMING_SNAKE_CASE      )
                 encoded_sequence += tokenizer.encode(SCREAMING_SNAKE_CASE		,     add_special_tokens=SCREAMING_SNAKE_CASE      )
                 A			:  Any   =		tokenizer.encode_plus(
                     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     add_special_tokens=SCREAMING_SNAKE_CASE		,     return_special_tokens_mask=SCREAMING_SNAKE_CASE		,     )
                 A			:  Optional[Any]   =		encoded_sequence_dict['''input_ids''']
                 A			:  int   =		encoded_sequence_dict['''special_tokens_mask''']
                 self.assertEqual(len(SCREAMING_SNAKE_CASE      )		,     len(SCREAMING_SNAKE_CASE      )      )

                 A			:  Tuple   =		[
                     (x if not special_tokens_mask[i] else None) for i, x in enumerate(SCREAMING_SNAKE_CASE      )
                 ]
                 A			:  Tuple   =		[x for x in filtered_sequence if x is not None]
                 self.assertEqual(SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )





@require_tokenizers
class 					A							(				unittest.TestCase		):





     def 							__lowerCAmelCase				(				self      )					->							List[Any]:






         """simple docstring"""






         A			:  Optional[Any]   =		AutoTokenizer.from_pretrained('''facebook/opt-350m'''		,     from_slow=SCREAMING_SNAKE_CASE      )
         A			:  Optional[Any]   =		'''A photo of a cat'''

         A			:  str   =		tokenizer.encode(
             SCREAMING_SNAKE_CASE		,     )
         self.assertEqual(SCREAMING_SNAKE_CASE		,     [2, 250, 1345, 9, 10, 4758]      )
         tokenizer.save_pretrained('''test_opt'''      )

         A			:  str   =		AutoTokenizer.from_pretrained('''./test_opt'''      )
         A			:  List[Any]   =		tokenizer.encode(
             SCREAMING_SNAKE_CASE		,     )
         self.assertEqual(SCREAMING_SNAKE_CASE		,     [2, 250, 1345, 9, 10, 4758]      )





     def 							__lowerCAmelCase				(				self      )					->							Tuple:






         """simple docstring"""






         A			:  Union[str, Any]   =		AutoTokenizer.from_pretrained('''facebook/opt-350m'''		,     use_slow=SCREAMING_SNAKE_CASE      )
         A			:  Dict   =		'''A photo of a cat'''

         A			:  Optional[int]   =		tokenizer.encode(
             SCREAMING_SNAKE_CASE		,     )
         # Same as above
         self.assertEqual(SCREAMING_SNAKE_CASE		,     [2, 250, 1345, 9, 10, 4758]      )





     @unittest.skip('''This test is failing because of a bug in the fast tokenizer'''      )
     def 							__lowerCAmelCase				(				self      )					->							Any:






         """simple docstring"""






         A			:  Optional[Any]   =		AutoTokenizer.from_pretrained('''facebook/opt-350m'''		,     from_slow=SCREAMING_SNAKE_CASE      )

         A			:  Dict   =		'''bos'''
         A			:  str   =		tokenizer.get_vocab()['''bos''']

         A			:  Dict   =		'''A photo of a cat'''
         A			:  List[Any]   =		tokenizer.encode(
             SCREAMING_SNAKE_CASE		,     )
         # We changed the bos token
         self.assertEqual(SCREAMING_SNAKE_CASE		,     [31957, 250, 1345, 9, 10, 4758]      )
         tokenizer.save_pretrained('''./tok'''      )
         A			:  Optional[Any]   =		AutoTokenizer.from_pretrained('''./tok'''      )
         self.assertTrue(tokenizer.is_fast      )
         A			:  Union[str, Any]   =		tokenizer.encode(
             SCREAMING_SNAKE_CASE		,     )
         self.assertEqual(SCREAMING_SNAKE_CASE		,     [31957, 250, 1345, 9, 10, 4758]      )






'''simple docstring'''




import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto import CONFIG_MAPPING


lowercase     :							Optional[Any]       						=     logging.get_logger(__name__)

lowercase     :							int       						=     {
    'ut/deta': 'https://huggingface.co/ut/deta/resolve/main/config.json',
}





class 					A							(				__snake_case		):
     __magic_name__          =						'''deta'''
     __magic_name__          =						{
         '''hidden_size''': '''d_model''',
         '''num_attention_heads''': '''encoder_attention_heads''',
     }





     def __init__(				self		,     SCREAMING_SNAKE_CASE=None		,     SCREAMING_SNAKE_CASE=900		,     SCREAMING_SNAKE_CASE=2048		,     SCREAMING_SNAKE_CASE=6		,     SCREAMING_SNAKE_CASE=2048		,     SCREAMING_SNAKE_CASE=8		,     SCREAMING_SNAKE_CASE=6		,     SCREAMING_SNAKE_CASE=1024		,     SCREAMING_SNAKE_CASE=8		,     SCREAMING_SNAKE_CASE=0.0		,     SCREAMING_SNAKE_CASE=True		,     SCREAMING_SNAKE_CASE="relu"		,     SCREAMING_SNAKE_CASE=256		,     SCREAMING_SNAKE_CASE=0.1		,     SCREAMING_SNAKE_CASE=0.0		,     SCREAMING_SNAKE_CASE=0.0		,     SCREAMING_SNAKE_CASE=0.02		,     SCREAMING_SNAKE_CASE=1.0		,     SCREAMING_SNAKE_CASE=True		,     SCREAMING_SNAKE_CASE=False		,     SCREAMING_SNAKE_CASE="sine"		,     SCREAMING_SNAKE_CASE=5		,     SCREAMING_SNAKE_CASE=4		,     SCREAMING_SNAKE_CASE=4		,     SCREAMING_SNAKE_CASE=True		,     SCREAMING_SNAKE_CASE=300		,     SCREAMING_SNAKE_CASE=True		,     SCREAMING_SNAKE_CASE=True		,     SCREAMING_SNAKE_CASE=1		,     SCREAMING_SNAKE_CASE=5		,     SCREAMING_SNAKE_CASE=2		,     SCREAMING_SNAKE_CASE=1		,     SCREAMING_SNAKE_CASE=1		,     SCREAMING_SNAKE_CASE=5		,     SCREAMING_SNAKE_CASE=2		,     SCREAMING_SNAKE_CASE=0.1		,     SCREAMING_SNAKE_CASE=0.25		,     **SCREAMING_SNAKE_CASE		,     )					->							Any:






         """simple docstring"""






         if backbone_config is None:
             logger.info('''`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone.'''      )
             A			:  Optional[Any]   =		CONFIG_MAPPING['''resnet'''](out_features=['''stage2''', '''stage3''', '''stage4''']      )
         else:
             if isinstance(SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      ):
                 A			:  List[Any]   =		backbone_config.pop('''model_type'''      )
                 A			:  List[Any]   =		CONFIG_MAPPING[backbone_model_type]
                 A			:  Optional[int]   =		config_class.from_dict(SCREAMING_SNAKE_CASE      )

         A			:  str   =		backbone_config
         A			:  Optional[int]   =		num_queries
         A			:  Dict   =		max_position_embeddings
         A			:  Optional[Any]   =		d_model
         A			:  Optional[Any]   =		encoder_ffn_dim
         A			:  List[str]   =		encoder_layers
         A			:  Tuple   =		encoder_attention_heads
         A			:  Optional[Any]   =		decoder_ffn_dim
         A			:  Optional[int]   =		decoder_layers
         A			:  List[str]   =		decoder_attention_heads
         A			:  Union[str, Any]   =		dropout
         A			:  str   =		attention_dropout
         A			:  Any   =		activation_dropout
         A			:  Optional[int]   =		activation_function
         A			:  Tuple   =		init_std
         A			:  Any   =		init_xavier_std
         A			:  Optional[Any]   =		encoder_layerdrop
         A			:  int   =		auxiliary_loss
         A			:  Dict   =		position_embedding_type
         # deformable attributes
         A			:  str   =		num_feature_levels
         A			:  Optional[int]   =		encoder_n_points
         A			:  Any   =		decoder_n_points
         A			:  Tuple   =		two_stage
         A			:  Dict   =		two_stage_num_proposals
         A			:  List[str]   =		with_box_refine
         A			:  List[str]   =		assign_first_stage
         if two_stage is True and with_box_refine is False:
             raise ValueError('''If two_stage is True, with_box_refine must be True.'''      )
         # Hungarian matcher
         A			:  Dict   =		class_cost
         A			:  Optional[int]   =		bbox_cost
         A			:  Optional[Any]   =		giou_cost
         # Loss coefficients
         A			:  int   =		mask_loss_coefficient
         A			:  int   =		dice_loss_coefficient
         A			:  Tuple   =		bbox_loss_coefficient
         A			:  int   =		giou_loss_coefficient
         A			:  Dict   =		eos_coefficient
         A			:  Optional[Any]   =		focal_alpha
         super().__init__(is_encoder_decoder=SCREAMING_SNAKE_CASE		,     **SCREAMING_SNAKE_CASE      )





     @property
     def 							__lowerCAmelCase				(				self      )					->							int:






         """simple docstring"""






         return self.encoder_attention_heads





     @property
     def 							__lowerCAmelCase				(				self      )					->							int:






         """simple docstring"""






         return self.d_model





     def 							__lowerCAmelCase				(				self      )					->							Any:






         """simple docstring"""






         A			:  Any   =		copy.deepcopy(self.__dict__      )
         A			:  Dict   =		self.backbone_config.to_dict()
         A			:  List[Any]   =		self.__class__.model_type
         return output




