




import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__    			=			get_tests_dir("""fixtures/spiece.model""")
@require_sentencepiece
@require_tokenizers
class 			a__    (					snake_case__ ,				unittest.TestCase      ):
						_a							:    Optional[Any]					=    DebertaVaTokenizer
						_a							:    Optional[Any]					=    DebertaVaTokenizerFast
						_a							:    List[str]					=    True
						_a							:    Optional[Any]					=    True






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											super().setUp()

											# We have a SentencePiece fixture for testing
											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			unk_token="<unk>"			)
											tokenizer.save_pretrained(self.tmpdirname			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "this is a test"
											__lowerCAmelCase				   =       "this is a test"
											return input_text, output_text






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "<pad>"
											__lowerCAmelCase				   =       0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A			)		,			_A			)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A			)		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       list(self.get_tokenizer().get_vocab().keys()			)
											self.assertEqual(vocab_keys[0]		,			"<pad>"			)
											self.assertEqual(vocab_keys[1]		,			"<unk>"			)
											self.assertEqual(vocab_keys[-1]		,			"[PAD]"			)
											self.assertEqual(len(_A			)		,			3_0_0_0_1			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											self.assertEqual(self.get_tokenizer().vocab_size		,			3_0_0_0_0			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       " \tHeLLo!how  \n Are yoU?  "
											__lowerCAmelCase				   =       ["▁hello", "!", "how", "▁are", "▁you", "?"]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."			)
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass






						@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."			)
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       " \tHeLLo!how  \n Are yoU?  "
											__lowerCAmelCase				   =       ["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
											# fmt: on

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			do_lower_case=_A		,			split_by_punct=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)

											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       self.get_tokenizer()
											__lowerCAmelCase				   =       self.get_rust_tokenizer()

											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."

											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A		,			add_special_tokens=_A			)			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       self.get_rust_tokenizer()
											__lowerCAmelCase				   =       tokenizer.encode(_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "This is a test"
											__lowerCAmelCase				   =       [1_3, 1, 4_3_9_8, 2_5, 2_1, 1_2_8_9]
											__lowerCAmelCase				   =       ["▁", "T", "his", "▁is", "▁a", "▁test"]
											__lowerCAmelCase				   =       ["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

											__lowerCAmelCase				   =       DebertaVaTokenizer(_A		,			keep_accents=_A			)
											__lowerCAmelCase				   =       DebertaVaTokenizerFast(_A		,			keep_accents=_A			)

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)

											# fmt: off
											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."
											__lowerCAmelCase				   =       [1_3, 1, 2_3, 3_8_6, 1_9, 5_6_1, 3_0_5_0, 1_5, 1_7, 4_8, 2_5, 8_2_5_6, 1_8, 1, 9]
											__lowerCAmelCase				   =       ["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
											__lowerCAmelCase				   =       ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
											# fmt: on

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)
											__lowerCAmelCase				   =       rust_tokenizer.convert_ids_to_tokens(_A			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       DebertaVaTokenizer(_A			)

											__lowerCAmelCase				   =       tokenizer.encode("sequence builders"			)
											__lowerCAmelCase				   =       tokenizer.encode("multi-sequence build"			)

											__lowerCAmelCase				   =       tokenizer.build_inputs_with_special_tokens(_A			)
											__lowerCAmelCase				   =       tokenizer.build_inputs_with_special_tokens(_A		,			_A			)

											self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]		,			_A			)
											self.assertEqual(
											    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]		,			_A		,			)






						@slow
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       {"input_ids": [[1, 3_9_8_6_7, 3_6, 1_9_3_9_0, 4_8_6, 2_7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 6_0_6_8_5, 1_2_2_5, 7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 9_3_6_7, 1_6_8_9_9, 1_8, 1_5_9_3_7, 5_3, 5_9_4, 7_7_3, 1_8, 1_6_2_8_7, 3_0_4_6_5, 3_6, 1_5_9_3_7, 6, 4_1_1_3_9, 3_8, 3_6_9_7_9, 6_0_7_6_3, 1_9_1, 6, 3_4_1_3_2, 9_9, 6, 5_0_5_3_8, 3_9_0, 4_3_2_3_0, 6, 3_4_1_3_2, 2_7_7_9, 2_0_8_5_0, 1_4, 6_9_9, 1_0_7_2, 1_1_9_4, 3_6, 3_8_2, 1_0_9_0_1, 5_3, 7, 6_9_9, 1_0_7_2, 2_0_8_4, 3_6, 2_0_4_2_2, 6_3_0, 5_3, 1_9, 1_0_5, 3_0_4_9, 1_8_9_6, 1_0_5_3, 1_6_8_9_9, 1_5_0_6, 1_1, 3_7_9_7_8, 4_2_4_3, 7, 1_2_3_7, 3_1_8_6_9, 2_0_0, 1_6_5_6_6, 6_5_4, 6, 3_5_0_5_2, 8_1_4_3_6, 7, 5_5_6_3_0, 1_3_5_9_3, 4, 2], [1, 2_6, 1_5_0_1_1, 1_3, 6_6_7, 8, 1_0_5_3, 1_8, 2_3_6_1_1, 1_2_3_7, 7_2_3_5_6, 1_2_8_2_0, 3_4, 1_0_4_1_3_4, 1_2_0_9, 3_5, 1_3_3_1_3, 6_6_2_7, 2_1, 2_0_2, 3_4_7, 7, 1_6_4, 2_3_9_9, 1_1, 4_6, 4_4_8_5, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_2_3_2, 2_8_6_4, 1_5_7_8_5, 1_4_9_5_1, 1_0_5, 5, 8_5_8_1, 1_2_5_0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=_A		,			model_name="microsoft/deberta-v2-xlarge"		,			revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"		,			)








'''simple docstring'''
from datetime import datetime as dt
import os

from github import Github


__snake_case							      =     [
    '''good first issue''',
    '''good second issue''',
    '''good difficult issue''',
    '''feature request''',
    '''new model''',
    '''wip''',
]


def 			a      (			)  ->	List[str]:



					'''simple docstring'''






					UpperCamelCase__   :List[str]												= Github(os.environ['''GITHUB_TOKEN''']   )
					UpperCamelCase__   :Tuple												= g.get_repo('''huggingface/transformers'''   )
					UpperCamelCase__   :Union[str, Any]												= repo.get_issues(state='''open'''   )

					for issue in open_issues:
										UpperCamelCase__   :List[Any]												= sorted([comment for comment in issue.get_comments()] ,					key=lambda __a   : i.created_at ,					reverse=__a   )
										UpperCamelCase__   :List[Any]												= comments[0] if len(__a   ) > 0 else None
										if (
										    last_comment is not None
										    and last_comment.user.login == "github-actions[bot]"
										    and (dt.utcnow() - issue.updated_at).days > 7
										    and (dt.utcnow() - issue.created_at).days >= 30
										    and not any(label.name.lower() in LABELS_TO_EXEMPT for label in issue.get_labels()   )
										):
															# print(f"Would close issue {issue.number} since it has been 7 days of inactivity since bot mention.")
															issue.edit(state='''closed'''   )
										elif (
										    (dt.utcnow() - issue.updated_at).days > 23
										    and (dt.utcnow() - issue.created_at).days >= 30
										    and not any(label.name.lower() in LABELS_TO_EXEMPT for label in issue.get_labels()   )
										):
															# print(f"Would add stale comment to {issue.number}")
															issue.create_comment(
															    '''This issue has been automatically marked as stale because it has not had '''
															    '''recent activity. If you think this still needs to be addressed '''
															    '''please comment on this thread.\n\nPlease note that issues that do not follow the '''
															    '''[contributing guidelines](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md) '''
															    '''are likely to be ignored.'''   )


if __name__ == "__main__":
		main()