
"""simple docstring"""






import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_							=					get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model')





@require_sentencepiece
@require_tokenizers
class 				snake_case    (			_UpperCamelCase				,    unittest.TestCase						):
   a_ :						Any			 =			GPTSwaTokenizer
   a_ :						int			 =			False
   a_ :						Union[str, Any]			 =			True
   a_ :						Any			 =			False







   def 					UpperCAmelCase__     (      self)					->Any:
      super().setUp()

      # We have a SentencePiece fixture for testing
      a_							=			GPTSwaTokenizer(__lowerCamelCase    ,				eos_token="<unk>"    ,				bos_token="<unk>"    ,				pad_token="<unk>")

      tokenizer.save_pretrained(self.tmpdirname)







   def 					UpperCAmelCase__     (      self    ,				__UpperCAmelCase)					->Any:
      a_							=			"This is a test"
      a_							=			"This is a test"
      return input_text, output_text







   def 					UpperCAmelCase__     (      self)					->List[str]:
      a_							=			"<s>"
      a_							=			1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase)    ,				__lowerCamelCase)
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase)    ,				__lowerCamelCase)







   def 					UpperCAmelCase__     (      self)					->int:
      a_							=			list(self.get_tokenizer().get_vocab().keys())

      self.assertEqual(vocab_keys[0]    ,				"<unk>")
      self.assertEqual(vocab_keys[1]    ,				"<s>")
      self.assertEqual(vocab_keys[-1]    ,				"j")
      self.assertEqual(len(__lowerCamelCase)    ,				20_00)







   def 					UpperCAmelCase__     (      self)					->Dict:
      self.assertEqual(self.get_tokenizer().vocab_size    ,				20_00)







   def 					UpperCAmelCase__     (      self)					->Union[str, Any]:
      a_							=			GPTSwaTokenizer(__lowerCamelCase)

      a_							=			tokenizer.tokenize("This is a test")
      self.assertListEqual(__lowerCamelCase    ,				["▁This", "▁is", "▁a", "▁t", "est"])

      self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase)    ,				[4_65, 2_87, 2_65, 6_31, 8_42])

      a_							=			tokenizer.tokenize("I was born in 92000, and this is falsé.")
      # fmt: off
      self.assertListEqual(
          __lowerCamelCase    ,				["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."]    ,				)
      # fmt: on

      a_							=			tokenizer.convert_tokens_to_ids(__lowerCamelCase)
      self.assertListEqual(
          __lowerCamelCase    ,				[2_62, 2_72, 15_25, 2_86, 2_71, 2_68, 60, 9_16, 6_33, 6_33, 6_33, 2_59, 2_66, 3_01, 2_87, 3_84, 3_67, 2_63, 1_98, 1_72, 2_60]    ,				)

      a_							=			tokenizer.convert_ids_to_tokens(__lowerCamelCase)
      # fmt: off
      self.assertListEqual(
          __lowerCamelCase    ,				["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."])
      # fmt: on







   def 					UpperCAmelCase__     (      self)					->str:
      a_							=			GPTSwaTokenizer(__lowerCamelCase)
      a_							=			["This is a test", "I was born in 92000, and this is falsé."]
      a_							=			[
          [4_65, 2_87, 2_65, 6_31, 8_42],
          [2_62, 2_72, 15_25, 2_86, 2_71, 2_68, 60, 9_16, 6_33, 6_33, 6_33, 2_59, 2_66, 3_01, 2_87, 3_84, 3_67, 2_63, 1_98, 1_72, 2_60],
      ]

      # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
      for text, expected_ids in zip(__lowerCamelCase    ,				__lowerCamelCase):
         self.assertListEqual(tokenizer.encode_fast(__lowerCamelCase)    ,				__lowerCamelCase)

      # Test that decode_fast returns the input text
      for text, token_ids in zip(__lowerCamelCase    ,				__lowerCamelCase):
         self.assertEqual(tokenizer.decode_fast(__lowerCamelCase)    ,				__lowerCamelCase)







   @slow
   def 					UpperCAmelCase__     (      self)					->List[Any]:
      a_							=			[
          "<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')",
          "Hey there, how are you doing this fine day?",
          "This is a text with a trailing spaces followed by a dot     .",
          "Häj sväjs lillebrör! =)",
          "Det är inget fel på Mr. Cool",
      ]

      # fmt: off
      a_							=			{"input_ids": [[6_34_23, 5, 68_11, 1_49_54, 2_82, 8_16, 38_21, 6_34_66, 6_34_25, 6_34_62, 18, 6_39_78, 6_78, 3_01, 13_20, 6_34_23, 6_34_55, 6_34_58, 18, 6_39_82, 42_46, 39_40, 19_01, 4_77_89, 55_47, 1_89_94], [1_96_30, 11_00, 6_34_46, 13_42, 6_33, 5_44, 44_88, 5_93, 51_02, 24_16, 6_34_95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [16_52, 4_28, 2_68, 19_36, 5_15, 2_68, 5_85_93, 2_24_13, 91_06, 5_46, 2_68, 3_32_13, 6_39_79, 6_98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_51_30, 6_34_50, 9_24, 6_34_49, 22_49, 40_62, 15_58, 3_18, 6_35_04, 2_14_98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_09, 3_77, 28_27, 25_59, 3_32, 65_75, 6_34_43, 2_68_01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
      # fmt: on
      self.tokenizer_integration_test_util(
          expected_encoding=__lowerCamelCase    ,				model_name="AI-Sweden/gpt-sw3-126m"    ,				sequences=__lowerCamelCase    ,				)





from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCAmelCase		:				Optional[Any]       			=		logging.get_logger(__name__)

__lowerCAmelCase		:				str       			=		{'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/config.json'}




class     snake_case__							(_UpperCamelCase   ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_       :							Dict							 =					"""openai-gpt"""
				SCREAMING_SNAKE_CASE_       :							Tuple							 =					{
				    """max_position_embeddings""": """n_positions""",
				    """hidden_size""": """n_embd""",
				    """num_attention_heads""": """n_head""",
				    """num_hidden_layers""": """n_layer""",
				}





				def __init__(      self				:     str					,      __lowerCamelCase				:     List[str]=4_04_78					,      __lowerCamelCase				:     List[Any]=5_12					,      __lowerCamelCase				:     List[str]=7_68					,      __lowerCamelCase				:     List[str]=12					,      __lowerCamelCase				:     Optional[Any]=12					,      __lowerCamelCase				:     str="gelu"					,      __lowerCamelCase				:     Optional[int]=0.1					,      __lowerCamelCase				:     Any=0.1					,      __lowerCamelCase				:     Dict=0.1					,      __lowerCamelCase				:     Any=1e-5					,      __lowerCamelCase				:     Optional[int]=0.02					,      __lowerCamelCase				:     Optional[int]="cls_index"					,      __lowerCamelCase				:     List[Any]=True					,      __lowerCamelCase				:     Optional[int]=None					,      __lowerCamelCase				:     Optional[Any]=True					,      __lowerCamelCase				:     Tuple=0.1					,      **__lowerCamelCase				:     Union[str, Any]					,      )					->     List[str]:
							a    =			vocab_size
							a    =			n_positions
							a    =			n_embd
							a    =			n_layer
							a    =			n_head
							a    =			afn
							a    =			resid_pdrop
							a    =			embd_pdrop
							a    =			attn_pdrop
							a    =			layer_norm_epsilon
							a    =			initializer_range
							a    =			summary_type
							a    =			summary_use_proj
							a    =			summary_activation
							a    =			summary_first_dropout
							a    =			summary_proj_to_labels
							super().__init__(**__lowerCamelCase							)






