


import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase		       =							get_tests_dir('''fixtures/test_sentencepiece.model''')





@require_sentencepiece
class   __a			(    __UpperCamelCase       ,       unittest.TestCase      ):
 __lowercase :							Union[str, Any]        = XLMProphetNetTokenizer
 __lowercase :							int        = False
 __lowercase :							Any        = True
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				Dict:




        '''simple docstring'''
        super().setUp()

        # We have a SentencePiece fixture for testing
        lowercase__:   List[str]  			=   XLMProphetNetTokenizer(lowerCAmelCase__				, keep_accents=lowerCAmelCase__ )
        tokenizer.save_pretrained(self.tmpdirname )
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				int:




        '''simple docstring'''
        lowercase__:   Any  			=   '[PAD]'
        lowercase__:   Dict  			=   0

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__ )				, lowerCAmelCase__ )
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__ )				, lowerCAmelCase__ )
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				Union[str, Any]:




        '''simple docstring'''
        lowercase__:   Optional[int]  			=   list(self.get_tokenizer().get_vocab().keys() )

        self.assertEqual(vocab_keys[0]				, '[PAD]' )
        self.assertEqual(vocab_keys[1]				, '[CLS]' )
        self.assertEqual(vocab_keys[-1]				, 'j' )
        self.assertEqual(len(lowerCAmelCase__ )				, 1_012 )
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				int:




        '''simple docstring'''
        self.assertEqual(self.get_tokenizer().vocab_size				, 1_012 )
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				Dict:




        '''simple docstring'''
        lowercase__:   int  			=   XLMProphetNetTokenizer(lowerCAmelCase__				, keep_accents=lowerCAmelCase__ )

        lowercase__:   Tuple  			=   tokenizer.tokenize('This is a test' )
        self.assertListEqual(lowerCAmelCase__				, ['▁This', '▁is', '▁a', '▁t', 'est'] )

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(lowerCAmelCase__ )				, [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]				, )

        lowercase__:   Tuple  			=   tokenizer.tokenize('I was born in 92000, and this is falsé.' )
        self.assertListEqual(
            lowerCAmelCase__				, [
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '9',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                'é',
                '.',
            ]				, )
        lowercase__:   Any  			=   tokenizer.convert_tokens_to_ids(lowerCAmelCase__ )
        self.assertListEqual(
            lowerCAmelCase__				, [
                value + tokenizer.fairseq_offset
                for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
            ]				, )

        lowercase__:   str  			=   tokenizer.convert_ids_to_tokens(lowerCAmelCase__ )
        self.assertListEqual(
            lowerCAmelCase__				, [
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '[UNK]',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                '[UNK]',
                '.',
            ]				, )
 @cached_property
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				List[str]:




        '''simple docstring'''
        return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased' )
 @slow
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				List[Any]:




        '''simple docstring'''
        lowercase__:   Union[str, Any]  			=   'Hello World!'
        lowercase__:   Union[str, Any]  			=   [35_389, 6_672, 49, 2]
        self.assertListEqual(lowerCAmelCase__				, self.big_tokenizer.encode(lowerCAmelCase__ ) )
 @slow
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				str:




        '''simple docstring'''
        lowercase__:   str  			=   {'input_ids': [[11_073, 82_783, 18, 26, 82_783, 549, 51_540, 248, 17_209, 1_301, 217, 20, 215_186, 1_325, 147, 17_209, 1_301, 217, 20, 56_370, 53, 122_020, 20, 16_477, 27, 87_355, 4_548, 20, 4_728, 78_392, 17, 159_969, 18, 26, 24_491, 629, 15, 538, 22_704, 5_439, 15, 2_788, 24_491, 9_885, 15, 43_534, 605, 15, 814, 18_403, 33_200, 29, 15, 43_534, 24_458, 12_410, 111, 24_966, 83_669, 9_637, 144_068, 26, 850, 22_346, 27, 147, 24_966, 83_669, 83_490, 26, 39_113, 735, 27, 689, 656, 2_800, 1_339, 4_600, 53, 122_020, 115_785, 34, 816, 1_339, 46_887, 18, 147, 53_905, 1_951, 42_238, 41_170, 17_732, 834, 436, 15, 27_523, 98_733, 217, 147, 5_542, 4_981, 930, 17_347, 16, 2], [20_091, 629, 94, 82_786, 58, 490, 20, 1_528, 84, 53_905, 344, 80_592, 110_128, 18_822, 5_267, 1_306, 62, 152_537, 308, 7_997, 401, 124_427, 549, 35_442, 225, 109, 15_055, 25_748, 147, 7_119, 43_712, 34, 767, 135_366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63_784, 119_466, 17, 147_808, 88_214, 18, 656, 81, 32, 3_296, 10_280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=lowerCAmelCase__				, model_name='microsoft/xprophetnet-large-wiki100-cased'				, revision='1acad1643ddd54a44df6a1b797ada8373685d90e'				, )




_lowerCamelCase       :				int       =			[4, 1, 7, 4, 2, 6, 4, 1, 5, 3, 7, 5]
_lowerCamelCase       :				str       =			[3, 7, 7, 4, 2, 6, 4, 1, 5, 3, 7, 5]
_lowerCamelCase       :				List[str]       =			{
    0: '''Sunday''',
    1: '''Monday''',
    2: '''Tuesday''',
    3: '''Wednesday''',
    4: '''Thursday''',
    5: '''Friday''',
    6: '''Saturday''',
}






def        a_		(       __lowercase   :	int       ,					__lowercase   :	int       ,					__lowercase   :	int ) ->  str:
			assert len(str(__lowercase ) ) > 2, "year should be in YYYY format"
			assert 1 <= month <= 12, "month should be between 1 to 12"
			assert 1 <= day <= 31, "day should be between 1 to 31"

			# Doomsday algorithm:
			_snake_case		=			year // 100
			_snake_case		=			(5 * (century % 4) + 2) % 7
			_snake_case		=			year % 100
			_snake_case		=			centurian % 12
			_snake_case		=			(
			    (centurian // 12) + centurian_m + (centurian_m // 4) + century_anchor
			) % 7
			_snake_case		=			(
			    DOOMSDAY_NOT_LEAP[month - 1]
			    if (year % 4 != 0) or (centurian == 0 and (year % 400) == 0)
			    else DOOMSDAY_LEAP[month - 1]
			)
			_snake_case		=			(dooms_day + day - day_anchor) % 7
			return WEEK_DAY_NAMES[week_day]


if __name__ == "__main__":
			import doctest

			doctest.testmod()