





'''simple docstring'''






import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class  _A (					__lowercase						,					unittest.TestCase		):
		lowercase__:   Optional[int]				     =   CodeGenTokenizer
		lowercase__:   str				     =   CodeGenTokenizerFast
		lowercase__:   Dict				     =   True
		lowercase__:   List[str]				     =   {'''add_prefix_space''': True}
		lowercase__:   Optional[int]				     =   False

		def 	lowercase__     (   self					:   Dict			)					->    Any:


					"""simple docstring"""

					super().setUp()

					# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
					__snake_case					:      Optional[int]							      =     [
					    """l""",
					    """o""",
					    """w""",
					    """e""",
					    """r""",
					    """s""",
					    """t""",
					    """i""",
					    """d""",
					    """n""",
					    """\u0120""",
					    """\u0120l""",
					    """\u0120n""",
					    """\u0120lo""",
					    """\u0120low""",
					    """er""",
					    """\u0120lowest""",
					    """\u0120newer""",
					    """\u0120wider""",
					    """<unk>""",
					    """<|endoftext|>""",
					]
					__snake_case					:      Any							      =     dict(zip(__magic_name__  ,      range(len(__magic_name__			)			)			)			)
					__snake_case					:      Union[str, Any]							      =     ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
					__snake_case					:      Union[str, Any]							      =     {"""unk_token""": """<unk>"""}

					__snake_case					:      Dict							      =     os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES["""vocab_file"""]			)
					__snake_case					:      Tuple							      =     os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES["""merges_file"""]			)
					with open(self.vocab_file  ,      """w"""  ,      encoding="""utf-8"""			) as fp:
								fp.write(json.dumps(__magic_name__			) + """\n"""			)
					with open(self.merges_file  ,      """w"""  ,      encoding="""utf-8"""			) as fp:
								fp.write("""\n""".join(__magic_name__			)			)

		def 	lowercase__     (   self					:   Dict  ,      **__magic_name__					:   Tuple			)					->    List[Any]:


					"""simple docstring"""

					kwargs.update(self.special_tokens_map			)
					return CodeGenTokenizer.from_pretrained(self.tmpdirname  ,      **__magic_name__			)

		def 	lowercase__     (   self					:   Any  ,      **__magic_name__					:   Union[str, Any]			)					->    List[Any]:


					"""simple docstring"""

					kwargs.update(self.special_tokens_map			)
					return CodeGenTokenizerFast.from_pretrained(self.tmpdirname  ,      **__magic_name__			)

		def 	lowercase__     (   self					:   int  ,      __magic_name__					:   Union[str, Any]			)					->    List[Any]:


					"""simple docstring"""

					__snake_case					:      int							      =     """lower newer"""
					__snake_case					:      Any							      =     """lower newer"""
					return input_text, output_text

		def 	lowercase__     (   self					:   List[str]			)					->    Optional[Any]:


					"""simple docstring"""

					__snake_case					:      Union[str, Any]							      =     CodeGenTokenizer(self.vocab_file  ,      self.merges_file  ,      **self.special_tokens_map			)
					__snake_case					:      List[Any]							      =     """lower newer"""
					__snake_case					:      int							      =     ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
					__snake_case					:      List[str]							      =     tokenizer.tokenize(__magic_name__  ,      add_prefix_space=__magic_name__			)
					self.assertListEqual(__magic_name__  ,      __magic_name__			)

					__snake_case					:      str							      =     tokens + [tokenizer.unk_token]
					__snake_case					:      Optional[int]							      =     [14, 15, 10, 9, 3, 2, 15, 19]
					self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__			)  ,      __magic_name__			)

		def 	lowercase__     (   self					:   Union[str, Any]			)					->    Union[str, Any]:


					"""simple docstring"""

					if not self.test_rust_tokenizer:
								return

					__snake_case					:      str							      =     self.get_tokenizer()
					__snake_case					:      int							      =     self.get_rust_tokenizer(add_prefix_space=__magic_name__			)

					__snake_case					:      List[Any]							      =     """lower newer"""

					# Testing tokenization
					__snake_case					:      int							      =     tokenizer.tokenize(__magic_name__  ,      add_prefix_space=__magic_name__			)
					__snake_case					:      Dict							      =     rust_tokenizer.tokenize(__magic_name__			)
					self.assertListEqual(__magic_name__  ,      __magic_name__			)

					# Testing conversion to ids without special tokens
					__snake_case					:      str							      =     tokenizer.encode(__magic_name__  ,      add_special_tokens=__magic_name__  ,      add_prefix_space=__magic_name__			)
					__snake_case					:      Any							      =     rust_tokenizer.encode(__magic_name__  ,      add_special_tokens=__magic_name__			)
					self.assertListEqual(__magic_name__  ,      __magic_name__			)

					# Testing conversion to ids with special tokens
					__snake_case					:      Optional[int]							      =     self.get_rust_tokenizer(add_prefix_space=__magic_name__			)
					__snake_case					:      List[str]							      =     tokenizer.encode(__magic_name__  ,      add_prefix_space=__magic_name__			)
					__snake_case					:      Union[str, Any]							      =     rust_tokenizer.encode(__magic_name__			)
					self.assertListEqual(__magic_name__  ,      __magic_name__			)

					# Testing the unknown token
					__snake_case					:      Dict							      =     tokens + [rust_tokenizer.unk_token]
					__snake_case					:      Any							      =     [14, 15, 10, 9, 3, 2, 15, 19]
					self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__magic_name__			)  ,      __magic_name__			)

		def 	lowercase__     (   self					:   Any  ,      *__magic_name__					:   str  ,      **__magic_name__					:   int			)					->    int:


					"""simple docstring"""

					pass

		def 	lowercase__     (   self					:   Union[str, Any]  ,      __magic_name__					:   Tuple=15			)					->    Any:


					"""simple docstring"""

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
											__snake_case					:      Any							      =     self.rust_tokenizer_class.from_pretrained(__magic_name__  ,      **__magic_name__			)

											# Simple input
											__snake_case					:      Any							      =     """This is a simple input"""
											__snake_case					:      int							      =     ["""This is a simple input 1""", """This is a simple input 2"""]
											__snake_case					:      int							      =     ("""This is a simple input""", """This is a pair""")
											__snake_case					:      Optional[int]							      =     [
											    ("""This is a simple input 1""", """This is a simple input 2"""),
											    ("""This is a simple pair 1""", """This is a simple pair 2"""),
											]

											# Simple input tests
											self.assertRaises(__magic_name__  ,      tokenizer_r.encode  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""			)

											# Simple input
											self.assertRaises(__magic_name__  ,      tokenizer_r.encode_plus  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""			)

											# Simple input
											self.assertRaises(
											    __magic_name__  ,      tokenizer_r.batch_encode_plus  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""  ,      )

											# Pair input
											self.assertRaises(__magic_name__  ,      tokenizer_r.encode  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""			)

											# Pair input
											self.assertRaises(__magic_name__  ,      tokenizer_r.encode_plus  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""			)

											# Pair input
											self.assertRaises(
											    __magic_name__  ,      tokenizer_r.batch_encode_plus  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""  ,      )

		def 	lowercase__     (   self					:   Optional[Any]			)					->    Union[str, Any]:


					"""simple docstring"""

					__snake_case					:      Tuple							      =     CodeGenTokenizer.from_pretrained(self.tmpdirname  ,      pad_token="""<pad>"""			)

					# Simple input
					__snake_case					:      Tuple							      =     """This is a simple input"""
					__snake_case					:      Union[str, Any]							      =     ["""This is a simple input looooooooong""", """This is a simple input"""]
					__snake_case					:      int							      =     ("""This is a simple input""", """This is a pair""")
					__snake_case					:      str							      =     [
					    ("""This is a simple input loooooong""", """This is a simple input"""),
					    ("""This is a simple pair loooooong""", """This is a simple pair"""),
					]

					__snake_case					:      Any							      =     tokenizer.pad_token_id

					__snake_case					:      List[str]							      =     tokenizer(__magic_name__  ,      padding="""max_length"""  ,      max_length=30  ,      return_tensors="""np"""			)
					__snake_case					:      List[Any]							      =     tokenizer(__magic_name__  ,      padding=__magic_name__  ,      truncate=__magic_name__  ,      return_tensors="""np"""			)
					__snake_case					:      Optional[Any]							      =     tokenizer(*__magic_name__  ,      padding="""max_length"""  ,      max_length=60  ,      return_tensors="""np"""			)
					__snake_case					:      Optional[Any]							      =     tokenizer(__magic_name__  ,      padding=__magic_name__  ,      truncate=__magic_name__  ,      return_tensors="""np"""			)

					# s
					# test single string max_length padding
					self.assertEqual(out_s["""input_ids"""].shape[-1]  ,      30			)
					self.assertTrue(pad_token_id in out_s["""input_ids"""]			)
					self.assertTrue(0 in out_s["""attention_mask"""]			)

					# s2
					# test automatic padding
					self.assertEqual(out_sa["""input_ids"""].shape[-1]  ,      33			)
					# long slice doesn't have padding
					self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]			)
					self.assertFalse(0 in out_sa["""attention_mask"""][0]			)
					# short slice does have padding
					self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]			)
					self.assertTrue(0 in out_sa["""attention_mask"""][1]			)

					# p
					# test single pair max_length padding
					self.assertEqual(out_p["""input_ids"""].shape[-1]  ,      60			)
					self.assertTrue(pad_token_id in out_p["""input_ids"""]			)
					self.assertTrue(0 in out_p["""attention_mask"""]			)

					# p2
					# test automatic padding pair
					self.assertEqual(out_pa["""input_ids"""].shape[-1]  ,      52			)
					# long slice pair doesn't have padding
					self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]			)
					self.assertFalse(0 in out_pa["""attention_mask"""][0]			)
					# short slice pair does have padding
					self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]			)
					self.assertTrue(0 in out_pa["""attention_mask"""][1]			)

		def 	lowercase__     (   self					:   Any			)					->    Dict:


					"""simple docstring"""

					__snake_case					:      Optional[Any]							      =     """$$$"""
					__snake_case					:      Union[str, Any]							      =     CodeGenTokenizer.from_pretrained(self.tmpdirname  ,      bos_token=__magic_name__  ,      add_bos_token=__magic_name__			)

					__snake_case					:      Tuple							      =     """This is a simple input"""
					__snake_case					:      Dict							      =     ["""This is a simple input 1""", """This is a simple input 2"""]

					__snake_case					:      int							      =     tokenizer.bos_token_id

					__snake_case					:      List[Any]							      =     tokenizer(__magic_name__			)
					__snake_case					:      str							      =     tokenizer(__magic_name__			)

					self.assertEqual(out_s.input_ids[0]  ,      __magic_name__			)
					self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids			)			)

					__snake_case					:      str							      =     tokenizer.decode(out_s.input_ids			)
					__snake_case					:      Tuple							      =     tokenizer.batch_decode(out_sa.input_ids			)

					self.assertEqual(decode_s.split()[0]  ,      __magic_name__			)
					self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa			)			)

		@slow
		def 	lowercase__     (   self					:   Union[str, Any]			)					->    int:


					"""simple docstring"""

					__snake_case					:      List[Any]							      =     CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""			)

					__snake_case					:      Dict							      =     """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
					__snake_case					:      Any							      =     """\nif len_a > len_b:      result = a\nelse:      result = b"""

					__snake_case					:      Dict							      =     tokenizer.encode(__magic_name__			)
					__snake_case					:      Any							      =     ["""^#""", re.escape("""<|endoftext|>"""			), """^'''""", """^\"\"\"""", """\n\n\n"""]
					__snake_case					:      Optional[int]							      =     tokenizer.decode(__magic_name__  ,      truncate_before_pattern=__magic_name__			)
					self.assertEqual(__magic_name__  ,      __magic_name__			)

		def 	lowercase__     (   self					:   Union[str, Any]			)					->    Union[str, Any]:


					"""simple docstring"""

					pass




'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase     :							List[str]       						=     logging.get_logger(__name__)

lowercase     :							Optional[int]       						=     {
    'google/switch-base-8': 'https://huggingface.co/google/switch-base-8/blob/main/config.json',
}





class 					A							(				__snake_case		):
     __magic_name__          =						'''switch_transformers'''
     __magic_name__          =						['''past_key_values''']
     __magic_name__          =						{'''hidden_size''': '''d_model''', '''num_attention_heads''': '''num_heads''', '''num_hidden_layers''': '''num_layers'''}





     def __init__(				self		,     SCREAMING_SNAKE_CASE=32128		,     SCREAMING_SNAKE_CASE=768		,     SCREAMING_SNAKE_CASE=64		,     SCREAMING_SNAKE_CASE=2048		,     SCREAMING_SNAKE_CASE=64		,     SCREAMING_SNAKE_CASE=12		,     SCREAMING_SNAKE_CASE=3		,     SCREAMING_SNAKE_CASE=12		,     SCREAMING_SNAKE_CASE=3		,     SCREAMING_SNAKE_CASE=12		,     SCREAMING_SNAKE_CASE=8		,     SCREAMING_SNAKE_CASE=False		,     SCREAMING_SNAKE_CASE=0.01		,     SCREAMING_SNAKE_CASE="float32"		,     SCREAMING_SNAKE_CASE=False		,     SCREAMING_SNAKE_CASE=32		,     SCREAMING_SNAKE_CASE=128		,     SCREAMING_SNAKE_CASE=0.1		,     SCREAMING_SNAKE_CASE=1e-6		,     SCREAMING_SNAKE_CASE=0.001		,     SCREAMING_SNAKE_CASE=0.001		,     SCREAMING_SNAKE_CASE=1.0		,     SCREAMING_SNAKE_CASE="relu"		,     SCREAMING_SNAKE_CASE=True		,     SCREAMING_SNAKE_CASE=False		,     SCREAMING_SNAKE_CASE=True		,     SCREAMING_SNAKE_CASE=0		,     SCREAMING_SNAKE_CASE=1		,     **SCREAMING_SNAKE_CASE		,     )					->							List[Any]:






         """simple docstring"""






         A			:  Union[str, Any]   =		vocab_size
         A			:  Any   =		d_model
         A			:  Dict   =		d_kv
         A			:  Optional[Any]   =		d_ff

         A			:  List[Any]   =		num_sparse_encoder_layers

         A			:  List[Any]   =		num_layers
         A			:  Optional[int]   =		(
             num_decoder_layers if num_decoder_layers is not None else self.num_layers
         )  # default = symmetry
         A			:  List[str]   =		num_sparse_decoder_layers

         # This tells us, each how many encoder layer we'll have to set a sparse layer.
         if self.num_sparse_encoder_layers > 0:
             A			:  int   =		self.num_layers // self.num_sparse_encoder_layers
         else:
             A			:  int   =		self.num_layers  # HACK: this will create 0 sparse layers

         # This tells us, each how many encoder layer we'll have to set a sparse layer.
         if self.num_sparse_decoder_layers > 0:
             A			:  str   =		self.num_decoder_layers // self.num_sparse_decoder_layers
         else:
             A			:  Dict   =		self.num_decoder_layers  # HACK: this will create 0 sparse layers

         A			:  List[Any]   =		num_heads
         A			:  str   =		num_experts
         A			:  Dict   =		expert_capacity
         A			:  Optional[int]   =		router_bias
         A			:  Any   =		router_jitter_noise
         if router_dtype not in ["float32", "float16", "bfloat16"]:
             raise ValueError(F'`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}'      )
         A			:  Dict   =		router_dtype

         A			:  int   =		router_ignore_padding_tokens
         A			:  List[Any]   =		relative_attention_num_buckets
         A			:  Dict   =		relative_attention_max_distance

         A			:  Optional[int]   =		dropout_rate
         A			:  Dict   =		layer_norm_epsilon
         A			:  Optional[int]   =		initializer_factor
         A			:  Union[str, Any]   =		feed_forward_proj
         A			:  Any   =		use_cache
         A			:  Tuple   =		add_router_probs

         A			:  List[str]   =		router_z_loss_coef
         A			:  str   =		router_aux_loss_coef

         A			:  Union[str, Any]   =		self.feed_forward_proj.split('''-'''      )
         A			:  Optional[Any]   =		act_info[-1]
         A			:  Union[str, Any]   =		act_info[0] == '''gated'''

         if len(SCREAMING_SNAKE_CASE      ) > 1 and act_info[0] != "gated" or len(SCREAMING_SNAKE_CASE      ) > 2:
             raise ValueError(
                 F'`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer.'
                 '''Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '''
                 '''\'gated-gelu\' or \'relu\''''      )

         # for backwards compatibility
         if feed_forward_proj == "gated-gelu":
             A			:  str   =		'''gelu_new'''

         super().__init__(
             pad_token_id=SCREAMING_SNAKE_CASE		,     eos_token_id=SCREAMING_SNAKE_CASE		,     is_encoder_decoder=SCREAMING_SNAKE_CASE		,     **SCREAMING_SNAKE_CASE		,     )




