



import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class 				__snake_case						(     UpperCamelCase_						,unittest.TestCase						):
							_a			      = BertTokenizer
							_a			      = BertTokenizerFast
							_a			      = True
							_a			      = True
							_a			      = filter_non_english




							def    UpperCAmelCase__			(       self     :       Optional[Any]):
									super().setUp()

									lowerCAmelCase_			:     Dict									= [
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''[PAD]''',
									    '''[MASK]''',
									    '''want''',
									    '''##want''',
									    '''##ed''',
									    '''wa''',
									    '''un''',
									    '''runn''',
									    '''##ing''',
									    ''',''',
									    '''low''',
									    '''lowest''',
									]
									lowerCAmelCase_			:     Optional[Any]									= os.path.join(self.tmpdirname				,				VOCAB_FILES_NAMES['''vocab_file'''])
									with open(self.vocab_file				,				'''w'''				,				encoding='''utf-8''') as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))




							def    UpperCAmelCase__			(       self     :       int				,				A_     :       Any):
									lowerCAmelCase_			:     str									= '''UNwant\u00E9d,running'''
									lowerCAmelCase_			:     Union[str, Any]									= '''unwanted, running'''
									return input_text, output_text




							def    UpperCAmelCase__			(       self     :       Union[str, Any]):
									lowerCAmelCase_			:     Optional[int]									= self.tokenizer_class(self.vocab_file)

									lowerCAmelCase_			:     Union[str, Any]									= tokenizer.tokenize('''UNwant\u00E9d,running''')
									self.assertListEqual(A_				,				['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''])
									self.assertListEqual(tokenizer.convert_tokens_to_ids(A_)				,				[9, 6, 7, 1_2, 1_0, 1_1])




							def    UpperCAmelCase__			(       self     :       int):
									if not self.test_rust_tokenizer:
											return

									lowerCAmelCase_			:     Tuple									= self.get_tokenizer()
									lowerCAmelCase_			:     str									= self.get_rust_tokenizer()

									lowerCAmelCase_			:     Optional[Any]									= '''UNwant\u00E9d,running'''

									lowerCAmelCase_			:     Any									= tokenizer.tokenize(A_)
									lowerCAmelCase_			:     Optional[Any]									= rust_tokenizer.tokenize(A_)
									self.assertListEqual(A_				,				A_)

									lowerCAmelCase_			:     str									= tokenizer.encode(A_				,				add_special_tokens=A_)
									lowerCAmelCase_			:     int									= rust_tokenizer.encode(A_				,				add_special_tokens=A_)
									self.assertListEqual(A_				,				A_)

									lowerCAmelCase_			:     Union[str, Any]									= self.get_rust_tokenizer()
									lowerCAmelCase_			:     Optional[int]									= tokenizer.encode(A_)
									lowerCAmelCase_			:     Optional[int]									= rust_tokenizer.encode(A_)
									self.assertListEqual(A_				,				A_)

									# With lower casing
									lowerCAmelCase_			:     Optional[Any]									= self.get_tokenizer(do_lower_case=A_)
									lowerCAmelCase_			:     Tuple									= self.get_rust_tokenizer(do_lower_case=A_)

									lowerCAmelCase_			:     Dict									= '''UNwant\u00E9d,running'''

									lowerCAmelCase_			:     Tuple									= tokenizer.tokenize(A_)
									lowerCAmelCase_			:     Any									= rust_tokenizer.tokenize(A_)
									self.assertListEqual(A_				,				A_)

									lowerCAmelCase_			:     Any									= tokenizer.encode(A_				,				add_special_tokens=A_)
									lowerCAmelCase_			:     List[str]									= rust_tokenizer.encode(A_				,				add_special_tokens=A_)
									self.assertListEqual(A_				,				A_)

									lowerCAmelCase_			:     List[str]									= self.get_rust_tokenizer()
									lowerCAmelCase_			:     int									= tokenizer.encode(A_)
									lowerCAmelCase_			:     Tuple									= rust_tokenizer.encode(A_)
									self.assertListEqual(A_				,				A_)




							def    UpperCAmelCase__			(       self     :       List[Any]):
									lowerCAmelCase_			:     List[Any]									= BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''')				,				['''ah''', '''\u535A''', '''\u63A8''', '''zz'''])




							def    UpperCAmelCase__			(       self     :       Optional[Any]):
									lowerCAmelCase_			:     List[Any]									= BasicTokenizer(do_lower_case=A_)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')				,				['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''])
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')				,				['''hello'''])




							def    UpperCAmelCase__			(       self     :       List[str]):
									lowerCAmelCase_			:     str									= BasicTokenizer(do_lower_case=A_				,				strip_accents=A_)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')				,				['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')				,				['''h\u00E9llo'''])




							def    UpperCAmelCase__			(       self     :       Optional[int]):
									lowerCAmelCase_			:     List[str]									= BasicTokenizer(do_lower_case=A_				,				strip_accents=A_)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')				,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')				,				['''hello'''])




							def    UpperCAmelCase__			(       self     :       Union[str, Any]):
									lowerCAmelCase_			:     Optional[Any]									= BasicTokenizer(do_lower_case=A_)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')				,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')				,				['''hello'''])




							def    UpperCAmelCase__			(       self     :       List[str]):
									lowerCAmelCase_			:     int									= BasicTokenizer(do_lower_case=A_)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')				,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])




							def    UpperCAmelCase__			(       self     :       Union[str, Any]):
									lowerCAmelCase_			:     Any									= BasicTokenizer(do_lower_case=A_				,				strip_accents=A_)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')				,				['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])




							def    UpperCAmelCase__			(       self     :       Dict):
									lowerCAmelCase_			:     Optional[Any]									= BasicTokenizer(do_lower_case=A_				,				strip_accents=A_)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')				,				['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])




							def    UpperCAmelCase__			(       self     :       List[str]):
									lowerCAmelCase_			:     List[str]									= BasicTokenizer(do_lower_case=A_				,				never_split=['''[UNK]'''])

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''')				,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''])




							def    UpperCAmelCase__			(       self     :       Any):
									lowerCAmelCase_			:     Dict									= BasicTokenizer()
									lowerCAmelCase_			:     str									= '''a\n\'ll !!to?\'d of, can\'t.'''
									lowerCAmelCase_			:     Dict									= ['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
									self.assertListEqual(tokenizer.tokenize(A_)				,				A_)




							def    UpperCAmelCase__			(       self     :       str):
									lowerCAmelCase_			:     Dict									= ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

									lowerCAmelCase_			:     Optional[Any]									= {}
									for i, token in enumerate(A_):
											lowerCAmelCase_			:     Union[str, Any]									= i
									lowerCAmelCase_			:     Optional[Any]									= WordpieceTokenizer(vocab=A_				,				unk_token='''[UNK]''')

									self.assertListEqual(tokenizer.tokenize('''''')				,				[])

									self.assertListEqual(tokenizer.tokenize('''unwanted running''')				,				['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''])

									self.assertListEqual(tokenizer.tokenize('''unwantedX running''')				,				['''[UNK]''', '''runn''', '''##ing'''])




							def    UpperCAmelCase__			(       self     :       str):
									self.assertTrue(_is_whitespace(''' '''))
									self.assertTrue(_is_whitespace('''\t'''))
									self.assertTrue(_is_whitespace('''\r'''))
									self.assertTrue(_is_whitespace('''\n'''))
									self.assertTrue(_is_whitespace('''\u00A0'''))

									self.assertFalse(_is_whitespace('''A'''))
									self.assertFalse(_is_whitespace('''-'''))




							def    UpperCAmelCase__			(       self     :       Tuple):
									self.assertTrue(_is_control('''\u0005'''))

									self.assertFalse(_is_control('''A'''))
									self.assertFalse(_is_control(''' '''))
									self.assertFalse(_is_control('''\t'''))
									self.assertFalse(_is_control('''\r'''))




							def    UpperCAmelCase__			(       self     :       List[str]):
									self.assertTrue(_is_punctuation('''-'''))
									self.assertTrue(_is_punctuation('''$'''))
									self.assertTrue(_is_punctuation('''`'''))
									self.assertTrue(_is_punctuation('''.'''))

									self.assertFalse(_is_punctuation('''A'''))
									self.assertFalse(_is_punctuation(''' '''))




							def    UpperCAmelCase__			(       self     :       Tuple):
									lowerCAmelCase_			:     Optional[int]									= self.get_tokenizer()
									lowerCAmelCase_			:     str									= self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(A_) for t in ['''Test''', '''\xad''', '''test''']]				,				[['''[UNK]'''], [], ['''[UNK]''']])

									self.assertListEqual(
									    [rust_tokenizer.tokenize(A_) for t in ['''Test''', '''\xad''', '''test''']]				,				[['''[UNK]'''], [], ['''[UNK]''']])




							@slow
							def    UpperCAmelCase__			(       self     :       Optional[Any]):
									lowerCAmelCase_			:     str									= self.tokenizer_class.from_pretrained('''bert-base-uncased''')

									lowerCAmelCase_			:     List[Any]									= tokenizer.encode('''sequence builders'''				,				add_special_tokens=A_)
									lowerCAmelCase_			:     str									= tokenizer.encode('''multi-sequence build'''				,				add_special_tokens=A_)

									lowerCAmelCase_			:     int									= tokenizer.build_inputs_with_special_tokens(A_)
									lowerCAmelCase_			:     int									= tokenizer.build_inputs_with_special_tokens(A_				,				A_)

									assert encoded_sentence == [1_0_1] + text + [1_0_2]
									assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




							def    UpperCAmelCase__			(       self     :       Any):
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
													lowerCAmelCase_			:     List[Any]									= self.rust_tokenizer_class.from_pretrained(A_				,				**A_)

													lowerCAmelCase_			:     Union[str, Any]									= F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
													lowerCAmelCase_			:     Tuple									= tokenizer_r.encode_plus(
													    A_				,				return_attention_mask=A_				,				return_token_type_ids=A_				,				return_offsets_mapping=A_				,				add_special_tokens=A_				,				)

													lowerCAmelCase_			:     Tuple									= tokenizer_r.do_lower_case if hasattr(A_				,				'''do_lower_case''') else False
													lowerCAmelCase_			:     List[str]									= (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''A'''),
													        ((1, 2), ''','''),
													        ((3, 5), '''na'''),
													        ((5, 6), '''##ï'''),
													        ((6, 8), '''##ve'''),
													        ((9, 1_5), tokenizer_r.mask_token),
													        ((1_6, 2_1), '''Allen'''),
													        ((2_1, 2_3), '''##NL'''),
													        ((2_3, 2_4), '''##P'''),
													        ((2_5, 3_3), '''sentence'''),
													        ((3_3, 3_4), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''a'''),
													        ((1, 2), ''','''),
													        ((3, 8), '''naive'''),
													        ((9, 1_5), tokenizer_r.mask_token),
													        ((1_6, 2_1), '''allen'''),
													        ((2_1, 2_3), '''##nl'''),
													        ((2_3, 2_4), '''##p'''),
													        ((2_5, 3_3), '''sentence'''),
													        ((3_3, 3_4), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]				,				tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']))
													self.assertEqual([e[0] for e in expected_results]				,				tokens['''offset_mapping'''])







							def    UpperCAmelCase__			(       self     :       Union[str, Any]):
									lowerCAmelCase_			:     Dict									= ['''的''', '''人''', '''有''']
									lowerCAmelCase_			:     str									= ''''''.join(A_)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
													lowerCAmelCase_			:     Tuple									= True
													lowerCAmelCase_			:     Optional[Any]									= self.tokenizer_class.from_pretrained(A_				,				**A_)
													lowerCAmelCase_			:     str									= self.rust_tokenizer_class.from_pretrained(A_				,				**A_)

													lowerCAmelCase_			:     Dict									= tokenizer_p.encode(A_				,				add_special_tokens=A_)
													lowerCAmelCase_			:     Union[str, Any]									= tokenizer_r.encode(A_				,				add_special_tokens=A_)

													lowerCAmelCase_			:     Optional[int]									= tokenizer_r.convert_ids_to_tokens(A_)
													lowerCAmelCase_			:     Optional[int]									= tokenizer_p.convert_ids_to_tokens(A_)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(A_				,				A_)
													self.assertListEqual(A_				,				A_)

													lowerCAmelCase_			:     int									= False
													lowerCAmelCase_			:     int									= self.rust_tokenizer_class.from_pretrained(A_				,				**A_)
													lowerCAmelCase_			:     List[str]									= self.tokenizer_class.from_pretrained(A_				,				**A_)

													lowerCAmelCase_			:     Tuple									= tokenizer_r.encode(A_				,				add_special_tokens=A_)
													lowerCAmelCase_			:     List[Any]									= tokenizer_p.encode(A_				,				add_special_tokens=A_)

													lowerCAmelCase_			:     Dict									= tokenizer_r.convert_ids_to_tokens(A_)
													lowerCAmelCase_			:     Union[str, Any]									= tokenizer_p.convert_ids_to_tokens(A_)

													# it is expected that only the first Chinese character is not preceded by "##".
													lowerCAmelCase_			:     int									= [
													    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(A_)
													]
													self.assertListEqual(A_				,				A_)
													self.assertListEqual(A_				,				A_)










from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


A__  :  List[str]								=					logging.get_logger(__name__)



class 				__snake_case						(     UpperCamelCase_						,UpperCamelCase_						):
							_a			      = '''maskformer-swin'''

							_a			      = {
							    '''num_attention_heads''': '''num_heads''',
							    '''num_hidden_layers''': '''num_layers''',
							}




							def __init__(       self     :       Union[str, Any]				,				A_     :       Dict=2_2_4				,				A_     :       Optional[Any]=4				,				A_     :       List[str]=3				,				A_     :       str=9_6				,				A_     :       Optional[Any]=[2, 2, 6, 2]				,				A_     :       Tuple=[3, 6, 1_2, 2_4]				,				A_     :       List[Any]=7				,				A_     :       List[Any]=4.0				,				A_     :       List[str]=True				,				A_     :       Dict=0.0				,				A_     :       int=0.0				,				A_     :       str=0.1				,				A_     :       Optional[int]="gelu"				,				A_     :       List[Any]=False				,				A_     :       int=0.02				,				A_     :       int=1e-5				,				A_     :       Optional[int]=None				,				A_     :       List[str]=None				,				**A_     :       List[Any]				,				):
									super().__init__(**A_)

									lowerCAmelCase_			:     Dict									= image_size
									lowerCAmelCase_			:     Optional[Any]									= patch_size
									lowerCAmelCase_			:     Tuple									= num_channels
									lowerCAmelCase_			:     Any									= embed_dim
									lowerCAmelCase_			:     List[str]									= depths
									lowerCAmelCase_			:     Union[str, Any]									= len(A_)
									lowerCAmelCase_			:     List[str]									= num_heads
									lowerCAmelCase_			:     Dict									= window_size
									lowerCAmelCase_			:     Optional[int]									= mlp_ratio
									lowerCAmelCase_			:     Dict									= qkv_bias
									lowerCAmelCase_			:     str									= hidden_dropout_prob
									lowerCAmelCase_			:     List[str]									= attention_probs_dropout_prob
									lowerCAmelCase_			:     Optional[int]									= drop_path_rate
									lowerCAmelCase_			:     Any									= hidden_act
									lowerCAmelCase_			:     str									= use_absolute_embeddings
									lowerCAmelCase_			:     List[str]									= layer_norm_eps
									lowerCAmelCase_			:     int									= initializer_range
									# we set the hidden_size attribute in order to make Swin work with VisionEncoderDecoderModel
									# this indicates the channel dimension after the last stage of the model
									lowerCAmelCase_			:     str									= int(embed_dim * 2 ** (len(A_) - 1))
									lowerCAmelCase_			:     Optional[Any]									= ['''stem'''] + [F"""stage{idx}""" for idx in range(1				,				len(A_) + 1)]
									lowerCAmelCase_					,  lowerCAmelCase_			:     int									= get_aligned_output_features_output_indices(
									    out_features=A_				,				out_indices=A_				,				stage_names=self.stage_names)





