
'''simple docstring'''





import warnings
from typing import List

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import is_flax_available, is_tf_available, is_torch_available


class        UpperCamelCase__     (			snake_case__):
 UpperCAmelCase__			:			List[Any]   =    ["""image_processor""", """tokenizer"""]
 UpperCAmelCase__			:			int   =    """OwlViTImageProcessor"""
 UpperCAmelCase__			:			Optional[Any]   =    ("""CLIPTokenizer""", """CLIPTokenizerFast""")






 def __init__(	self     :Tuple		, _A     :List[str]=None		, _A     :Optional[Any]=None		, **_A     :List[Any]	) ->	int:







       '''simple docstring'''



       __A						=	None
       if "feature_extractor" in kwargs:
             warnings.warn(
                 'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
                 ' instead.'		, _A		, )
             __A						=	kwargs.pop('feature_extractor'	)

       __A						=	image_processor if image_processor is not None else feature_extractor
       if image_processor is None:
             raise ValueError('You need to specify an `image_processor`.'	)
       if tokenizer is None:
             raise ValueError('You need to specify a `tokenizer`.'	)

       super().__init__(_A		, _A	)






 def __call__(	self     :Any		, _A     :int=None		, _A     :Any=None		, _A     :Dict=None		, _A     :int="max_length"		, _A     :List[str]="np"		, **_A     :Any	) ->	Optional[Any]:







       '''simple docstring'''



       if text is None and query_images is None and images is None:
             raise ValueError(
                 'You have to specify at least one text or query image or image. All three cannot be none.'	)

       if text is not None:
             if isinstance(_A		, _A	) or (isinstance(_A		, _A	) and not isinstance(text[0]		, _A	)):
                   __A						=	[self.tokenizer(_A		, padding=_A		, return_tensors=_A		, **_A	)]

             elif isinstance(_A		, _A	) and isinstance(text[0]		, _A	):
                   __A						=	[]

                   # Maximum number of queries across batch
                   __A						=	max([len(_A	) for t in text]	)

                   # Pad all batch samples to max number of text queries
                   for t in text:
                         if len(_A	) != max_num_queries:
                               __A						=	t + [' '] * (max_num_queries - len(_A	))

                         __A						=	self.tokenizer(_A		, padding=_A		, return_tensors=_A		, **_A	)
                         encodings.append(_A	)
             else:
                   raise TypeError('Input text should be a string, a list of strings or a nested list of strings'	)

             if return_tensors == "np":
                   __A						=	np.concatenate([encoding['input_ids'] for encoding in encodings]		, axis=0	)
                   __A						=	np.concatenate([encoding['attention_mask'] for encoding in encodings]		, axis=0	)

             elif return_tensors == "jax" and is_flax_available():
                   import jax.numpy as jnp

                   __A						=	jnp.concatenate([encoding['input_ids'] for encoding in encodings]		, axis=0	)
                   __A						=	jnp.concatenate([encoding['attention_mask'] for encoding in encodings]		, axis=0	)

             elif return_tensors == "pt" and is_torch_available():
                   import torch

                   __A						=	torch.cat([encoding['input_ids'] for encoding in encodings]		, dim=0	)
                   __A						=	torch.cat([encoding['attention_mask'] for encoding in encodings]		, dim=0	)

             elif return_tensors == "tf" and is_tf_available():
                   import tensorflow as tf

                   __A						=	tf.stack([encoding['input_ids'] for encoding in encodings]		, axis=0	)
                   __A						=	tf.stack([encoding['attention_mask'] for encoding in encodings]		, axis=0	)

             else:
                   raise ValueError('Target return tensor type could not be returned'	)

             __A						=	BatchEncoding()
             __A						=	input_ids
             __A						=	attention_mask

       if query_images is not None:
             __A						=	BatchEncoding()
             __A						=	self.image_processor(
                 _A		, return_tensors=_A		, **_A	).pixel_values
             __A						=	query_pixel_values

       if images is not None:
             __A						=	self.image_processor(_A		, return_tensors=_A		, **_A	)

       if text is not None and images is not None:
             __A						=	image_features.pixel_values
             return encoding
       elif query_images is not None and images is not None:
             __A						=	image_features.pixel_values
             return encoding
       elif text is not None or query_images is not None:
             return encoding
       else:
             return BatchEncoding(data=dict(**_A	)		, tensor_type=_A	)






 def 		lowercase_  (	self     :str		, *_A     :str		, **_A     :Tuple	) ->	Tuple:







       '''simple docstring'''



       return self.image_processor.post_process(*_A		, **_A	)






 def 		lowercase_  (	self     :List[str]		, *_A     :int		, **_A     :Dict	) ->	Any:







       '''simple docstring'''



       return self.image_processor.post_process_object_detection(*_A		, **_A	)






 def 		lowercase_  (	self     :int		, *_A     :Tuple		, **_A     :int	) ->	Any:







       '''simple docstring'''



       return self.image_processor.post_process_image_guided_detection(*_A		, **_A	)






 def 		lowercase_  (	self     :Any		, *_A     :Dict		, **_A     :List[str]	) ->	Union[str, Any]:







       '''simple docstring'''



       return self.tokenizer.batch_decode(*_A		, **_A	)






 def 		lowercase_  (	self     :int		, *_A     :Dict		, **_A     :Union[str, Any]	) ->	Optional[int]:







       '''simple docstring'''



       return self.tokenizer.decode(*_A		, **_A	)






 @property
 def 		lowercase_  (	self     :List[str]	) ->	Optional[Any]:







       '''simple docstring'''



       warnings.warn(
           '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'		, _A		, )
       return self.image_processor_class






 @property
 def 		lowercase_  (	self     :Union[str, Any]	) ->	Optional[Any]:







       '''simple docstring'''



       warnings.warn(
           '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'		, _A		, )
       return self.image_processor












import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__    			=			get_tests_dir("""fixtures/spiece.model""")
@require_sentencepiece
@require_tokenizers
class 			a__    (					snake_case__ ,				unittest.TestCase      ):
						_a							:    Optional[Any]					=    DebertaVaTokenizer
						_a							:    Optional[Any]					=    DebertaVaTokenizerFast
						_a							:    List[str]					=    True
						_a							:    Optional[Any]					=    True






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											super().setUp()

											# We have a SentencePiece fixture for testing
											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			unk_token="<unk>"			)
											tokenizer.save_pretrained(self.tmpdirname			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "this is a test"
											__lowerCAmelCase				   =       "this is a test"
											return input_text, output_text






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "<pad>"
											__lowerCAmelCase				   =       0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A			)		,			_A			)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A			)		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       list(self.get_tokenizer().get_vocab().keys()			)
											self.assertEqual(vocab_keys[0]		,			"<pad>"			)
											self.assertEqual(vocab_keys[1]		,			"<unk>"			)
											self.assertEqual(vocab_keys[-1]		,			"[PAD]"			)
											self.assertEqual(len(_A			)		,			3_0_0_0_1			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											self.assertEqual(self.get_tokenizer().vocab_size		,			3_0_0_0_0			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       " \tHeLLo!how  \n Are yoU?  "
											__lowerCAmelCase				   =       ["▁hello", "!", "how", "▁are", "▁you", "?"]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."			)
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass






						@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."			)
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       " \tHeLLo!how  \n Are yoU?  "
											__lowerCAmelCase				   =       ["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       self.get_tokenizer()
											__lowerCAmelCase				   =       self.get_rust_tokenizer()

											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."

											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       self.get_rust_tokenizer()
											__lowerCAmelCase				   =       tokenizer.encode(_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "This is a test"
											__lowerCAmelCase				   =       [1_3, 1, 4_3_9_8, 2_5, 2_1, 1_2_8_9]
											__lowerCAmelCase				   =       ["▁", "T", "his", "▁is", "▁a", "▁test"]
											__lowerCAmelCase				   =       ["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			keep_accents=_A			)
											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			keep_accents=_A			)

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)

											# fmt: off
											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       [1_3, 1, 2_3, 3_8_6, 1_9, 5_6_1, 3_0_5_0, 1_5, 1_7, 4_8, 2_5, 8_2_5_6, 1_8, 1, 9]
											__lowerCAmelCase				   =       ["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
											__lowerCAmelCase				   =       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       DebertaVaTokenizer(_A			)

											__lowerCAmelCase				   =       tokenizer.encode("sequence builders"			)
											__lowerCAmelCase				   =       tokenizer.encode("multi-sequence build"			)

											__lowerCAmelCase				   =       tokenizer.build_inputs_with_special_tokens(_A			)
											__lowerCAmelCase				   =       tokenizer.build_inputs_with_special_tokens(_A		,			_A			)

											self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]		,			_A			)
											self.assertEqual(
											    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]		,			_A		,			)






						@slow
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       {"input_ids": [[1, 3_9_8_6_7, 3_6, 1_9_3_9_0, 4_8_6, 2_7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 6_0_6_8_5, 1_2_2_5, 7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 9_3_6_7, 1_6_8_9_9, 1_8, 1_5_9_3_7, 5_3, 5_9_4, 7_7_3, 1_8, 1_6_2_8_7, 3_0_4_6_5, 3_6, 1_5_9_3_7, 6, 4_1_1_3_9, 3_8, 3_6_9_7_9, 6_0_7_6_3, 1_9_1, 6, 3_4_1_3_2, 9_9, 6, 5_0_5_3_8, 3_9_0, 4_3_2_3_0, 6, 3_4_1_3_2, 2_7_7_9, 2_0_8_5_0, 1_4, 6_9_9, 1_0_7_2, 1_1_9_4, 3_6, 3_8_2, 1_0_9_0_1, 5_3, 7, 6_9_9, 1_0_7_2, 2_0_8_4, 3_6, 2_0_4_2_2, 6_3_0, 5_3, 1_9, 1_0_5, 3_0_4_9, 1_8_9_6, 1_0_5_3, 1_6_8_9_9, 1_5_0_6, 1_1, 3_7_9_7_8, 4_2_4_3, 7, 1_2_3_7, 3_1_8_6_9, 2_0_0, 1_6_5_6_6, 6_5_4, 6, 3_5_0_5_2, 8_1_4_3_6, 7, 5_5_6_3_0, 1_3_5_9_3, 4, 2], [1, 2_6, 1_5_0_1_1, 1_3, 6_6_7, 8, 1_0_5_3, 1_8, 2_3_6_1_1, 1_2_3_7, 7_2_3_5_6, 1_2_8_2_0, 3_4, 1_0_4_1_3_4, 1_2_0_9, 3_5, 1_3_3_1_3, 6_6_2_7, 2_1, 2_0_2, 3_4_7, 7, 1_6_4, 2_3_9_9, 1_1, 4_6, 4_4_8_5, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_2_3_2, 2_8_6_4, 1_5_7_8_5, 1_4_9_5_1, 1_0_5, 5, 8_5_8_1, 1_2_5_0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=_A		,			model_name="microsoft/deberta-v2-xlarge"		,			revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"		,			)




