



import unittest

from transformers import BertGenerationTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCAmelCase          =       '''▁'''

__UpperCAmelCase          =       get_tests_dir('''fixtures/test_sentencepiece.model''')



@require_sentencepiece
class 				lowerCAmelCase_		(  a__       ,    unittest.TestCase							):
       UpperCAmelCase__				:					str        =  BertGenerationTokenizer
       UpperCAmelCase__				:					Tuple        =  False
       UpperCAmelCase__				:					Optional[Any]        =  True
       def 					snake_case_       (			self       )     ->		int:
          super().setUp()

          UpperCamelCase    :  List[str]							=	BertGenerationTokenizer(SCREAMING_SNAKE_CASE_,			keep_accents=SCREAMING_SNAKE_CASE_       )
          tokenizer.save_pretrained(self.tmpdirname       )
       def 					snake_case_       (			self       )     ->		Dict:
          UpperCamelCase    :  str							=	'<s>'
          UpperCamelCase    :  Any							=	1

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE_       ),			SCREAMING_SNAKE_CASE_       )
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE_       ),			SCREAMING_SNAKE_CASE_       )
       def 					snake_case_       (			self       )     ->		str:
          UpperCamelCase    :  Dict							=	list(self.get_tokenizer().get_vocab().keys()       )

          self.assertEqual(vocab_keys[0],			'<unk>'       )
          self.assertEqual(vocab_keys[1],			'<s>'       )
          self.assertEqual(vocab_keys[-1],			'<pad>'       )
          self.assertEqual(len(SCREAMING_SNAKE_CASE_       ),			1002       )
       def 					snake_case_       (			self       )     ->		Tuple:
          self.assertEqual(self.get_tokenizer().vocab_size,			1000       )
       def 					snake_case_       (			self       )     ->		Optional[int]:
          UpperCamelCase    :  Optional[Any]							=	BertGenerationTokenizer(SCREAMING_SNAKE_CASE_,			keep_accents=SCREAMING_SNAKE_CASE_       )

          UpperCamelCase    :  Union[str, Any]							=	tokenizer.tokenize('This is a test'       )
          self.assertListEqual(SCREAMING_SNAKE_CASE_,			['▁This', '▁is', '▁a', '▁t', 'est']       )

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_       ),			[285, 46, 10, 170, 382],			)

          UpperCamelCase    :  Tuple							=	tokenizer.tokenize('I was born in 92000, and this is falsé.'       )
          self.assertListEqual(
              SCREAMING_SNAKE_CASE_,			[
                  SPIECE_UNDERLINE + 'I',
                  SPIECE_UNDERLINE + 'was',
                  SPIECE_UNDERLINE + 'b',
                  'or',
                  'n',
                  SPIECE_UNDERLINE + 'in',
                  SPIECE_UNDERLINE + '',
                  '9',
                  '2',
                  '0',
                  '0',
                  '0',
                  ',',
                  SPIECE_UNDERLINE + 'and',
                  SPIECE_UNDERLINE + 'this',
                  SPIECE_UNDERLINE + 'is',
                  SPIECE_UNDERLINE + 'f',
                  'al',
                  's',
                  'é',
                  '.',
              ],			)
          UpperCamelCase    :  List[Any]							=	tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_       )
          self.assertListEqual(
              SCREAMING_SNAKE_CASE_,			[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4],			)

          UpperCamelCase    :  List[Any]							=	tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_       )
          self.assertListEqual(
              SCREAMING_SNAKE_CASE_,			[
                  SPIECE_UNDERLINE + 'I',
                  SPIECE_UNDERLINE + 'was',
                  SPIECE_UNDERLINE + 'b',
                  'or',
                  'n',
                  SPIECE_UNDERLINE + 'in',
                  SPIECE_UNDERLINE + '',
                  '<unk>',
                  '2',
                  '0',
                  '0',
                  '0',
                  ',',
                  SPIECE_UNDERLINE + 'and',
                  SPIECE_UNDERLINE + 'this',
                  SPIECE_UNDERLINE + 'is',
                  SPIECE_UNDERLINE + 'f',
                  'al',
                  's',
                  '<unk>',
                  '.',
              ],			)
       @cached_property
       def 					snake_case_       (			self       )     ->		str:
          return BertGenerationTokenizer.from_pretrained('google/bert_for_seq_generation_L-24_bbc_encoder'       )
       @slow
       def 					snake_case_       (			self       )     ->		Union[str, Any]:
          UpperCamelCase    :  Dict							=	'Hello World!'
          UpperCamelCase    :  Dict							=	[1_8536, 2260, 101]

          self.assertListEqual(SCREAMING_SNAKE_CASE_,			self.big_tokenizer.encode(SCREAMING_SNAKE_CASE_       )       )
       @slow
       def 					snake_case_       (			self       )     ->		Tuple:
          UpperCamelCase    :  List[str]							=	(
              'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
              ' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'
          )
          UpperCamelCase    :  Optional[int]							=	[
              871,
              419,
              358,
              946,
              991,
              2521,
              452,
              358,
              1357,
              387,
              7751,
              3536,
              112,
              985,
              456,
              126,
              865,
              938,
              5400,
              5734,
              458,
              1368,
              467,
              786,
              2462,
              5246,
              1159,
              633,
              865,
              4519,
              457,
              582,
              852,
              2557,
              427,
              916,
              508,
              405,
              3_4324,
              497,
              391,
              408,
              1_1342,
              1244,
              385,
              100,
              938,
              985,
              456,
              574,
              362,
              1_2597,
              3200,
              3129,
              1172,
          ]

          self.assertListEqual(SCREAMING_SNAKE_CASE_,			self.big_tokenizer.encode(SCREAMING_SNAKE_CASE_       )       )
       @require_torch
       @slow
       def 					snake_case_       (			self       )     ->		str:
          import torch

          from transformers import BertGenerationConfig, BertGenerationEncoder

          # Build sequence
          UpperCamelCase    :  Optional[Any]							=	list(self.big_tokenizer.get_vocab().keys()       )[:10]
          UpperCamelCase    :  Optional[Any]							=	' '.join(SCREAMING_SNAKE_CASE_       )
          UpperCamelCase    :  Optional[Any]							=	self.big_tokenizer.encode_plus(SCREAMING_SNAKE_CASE_,			return_tensors='pt',			return_token_type_ids=SCREAMING_SNAKE_CASE_       )
          UpperCamelCase    :  Dict							=	self.big_tokenizer.batch_encode_plus(
              [sequence + ' ' + sequence],			return_tensors='pt',			return_token_type_ids=SCREAMING_SNAKE_CASE_       )

          UpperCamelCase    :  Optional[Any]							=	BertGenerationConfig()
          UpperCamelCase    :  Tuple							=	BertGenerationEncoder(SCREAMING_SNAKE_CASE_       )

          assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

          with torch.no_grad():
             model(**SCREAMING_SNAKE_CASE_       )
             model(**SCREAMING_SNAKE_CASE_       )






       @slow
       def 					snake_case_       (			self       )     ->		Union[str, Any]:
          # fmt: off
          UpperCamelCase    :  Union[str, Any]							=	{'input_ids': [[3_9286, 458, 3_6335, 2001, 456, 1_3073, 1_3266, 455, 113, 7746, 1741, 1_1157, 391, 1_3073, 1_3266, 455, 113, 3967, 3_5412, 113, 4936, 109, 3870, 2377, 113, 3_0084, 4_5720, 458, 134, 1_7496, 112, 503, 1_1672, 113, 118, 112, 5665, 1_3347, 3_8687, 112, 1496, 3_1389, 112, 3268, 4_7264, 134, 962, 112, 1_6377, 8035, 2_3130, 430, 1_2169, 1_5518, 2_8592, 458, 146, 4_1697, 109, 391, 1_2169, 1_5518, 1_6689, 458, 146, 4_1358, 109, 452, 726, 4034, 111, 763, 3_5412, 5082, 388, 1903, 111, 9051, 391, 2870, 4_8918, 1900, 1123, 550, 998, 112, 9586, 1_5985, 455, 391, 410, 2_2955, 3_7636, 114], [448, 1_7496, 419, 3663, 385, 763, 113, 2_7533, 2870, 3283, 1_3043, 1639, 2_4713, 523, 656, 2_4013, 1_8550, 2521, 517, 2_7014, 2_1244, 420, 1212, 1465, 391, 927, 4833, 388, 578, 1_1786, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [484, 2169, 7687, 2_1932, 1_8146, 726, 363, 1_7032, 3391, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=SCREAMING_SNAKE_CASE_,			model_name='google/bert_for_seq_generation_L-24_bbc_encoder',			revision='c817d1fd1be2ffa69431227a1fe320544943d4db',			)








def 		UpperCamelCase	(   snake_case__       :    int       ,     snake_case__       :    int	)	->     int:
   return int(input_a == input_a == 0	)
def 		UpperCamelCase	(   )	->     None:
   print('Truth Table of NOR Gate:'	)
   print('|   Input 1   |  Input 2  | Output |'	)
   print(F"""|      0      |    0      |  {nor_gate(0       ,     0	)}     |"""	)
   print(F"""|      0      |    1      |  {nor_gate(0       ,     1	)}     |"""	)
   print(F"""|      1      |    0      |  {nor_gate(1       ,     0	)}     |"""	)
   print(F"""|      1      |    1      |  {nor_gate(1       ,     1	)}     |"""	)


if __name__ == "__main__":
    import doctest

    doctest.testmod()
    main()



