





import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class     __snake_case       (							__lowerCamelCase   ,			unittest.TestCase			):





       '''simple docstring'''







       lowerCAmelCase__      		=     CodeGenTokenizer
       lowerCAmelCase__      		=     CodeGenTokenizerFast
       lowerCAmelCase__      		=     True
       lowerCAmelCase__      		=     {"""add_prefix_space""": True}
       lowerCAmelCase__      		=     False




       def 							UpperCAmelCase__ (  self :    Optional[int]		):
         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         __snake_case:						List[Any]   =						[
             """l""",
             """o""",
             """w""",
             """e""",
             """r""",
             """s""",
             """t""",
             """i""",
             """d""",
             """n""",
             """\u0120""",
             """\u0120l""",
             """\u0120n""",
             """\u0120lo""",
             """\u0120low""",
             """er""",
             """\u0120lowest""",
             """\u0120newer""",
             """\u0120wider""",
             """<unk>""",
             """<|endoftext|>""",
         ]
         __snake_case:						Tuple   =						dict(zip(A							,				range(len(A		)		)		)		)
         __snake_case:						Union[str, Any]   =						["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
         __snake_case:						List[str]   =						{"""unk_token""": """<unk>"""}

         __snake_case:						List[Any]   =						os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
         __snake_case:						Optional[int]   =						os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""merges_file"""]		)
         with open(self.vocab_file							,				"""w"""							,				encoding="""utf-8"""		) as fp:
           fp.write(json.dumps(A		) + """\n"""		)
         with open(self.merges_file							,				"""w"""							,				encoding="""utf-8"""		) as fp:
           fp.write("""\n""".join(A		)		)




       def 							UpperCAmelCase__ (  self :    Tuple							,				**A :    Optional[int]		):
         kwargs.update(self.special_tokens_map		)
         return CodeGenTokenizer.from_pretrained(self.tmpdirname							,				**A		)




       def 							UpperCAmelCase__ (  self :    Dict							,				**A :    int		):
         kwargs.update(self.special_tokens_map		)
         return CodeGenTokenizerFast.from_pretrained(self.tmpdirname							,				**A		)




       def 							UpperCAmelCase__ (  self :    Dict							,				A :    List[Any]		):
         __snake_case:						int   =						"""lower newer"""
         __snake_case:						List[str]   =						"""lower newer"""
         return input_text, output_text




       def 							UpperCAmelCase__ (  self :    Tuple		):
         __snake_case:						Union[str, Any]   =						CodeGenTokenizer(self.vocab_file							,				self.merges_file							,				**self.special_tokens_map		)
         __snake_case:						int   =						"""lower newer"""
         __snake_case:						List[str]   =						["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
         __snake_case:						Optional[int]   =						tokenizer.tokenize(A							,				add_prefix_space=A		)
         self.assertListEqual(A							,				A		)

         __snake_case:						Tuple   =						tokens + [tokenizer.unk_token]
         __snake_case:						Tuple   =						[14, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(A		)							,				A		)




       def 							UpperCAmelCase__ (  self :    int		):
         if not self.test_rust_tokenizer:
           return

         __snake_case:						List[str]   =						self.get_tokenizer()
         __snake_case:						List[Any]   =						self.get_rust_tokenizer(add_prefix_space=A		)

         __snake_case:						int   =						"""lower newer"""

         # Testing tokenization
         __snake_case:						Dict   =						tokenizer.tokenize(A							,				add_prefix_space=A		)
         __snake_case:						int   =						rust_tokenizer.tokenize(A		)
         self.assertListEqual(A							,				A		)

         # Testing conversion to ids without special tokens
         __snake_case:						List[Any]   =						tokenizer.encode(A							,				add_special_tokens=A							,				add_prefix_space=A		)
         __snake_case:						Union[str, Any]   =						rust_tokenizer.encode(A							,				add_special_tokens=A		)
         self.assertListEqual(A							,				A		)

         # Testing conversion to ids with special tokens
         __snake_case:						int   =						self.get_rust_tokenizer(add_prefix_space=A		)
         __snake_case:						Optional[Any]   =						tokenizer.encode(A							,				add_prefix_space=A		)
         __snake_case:						Optional[Any]   =						rust_tokenizer.encode(A		)
         self.assertListEqual(A							,				A		)

         # Testing the unknown token
         __snake_case:						str   =						tokens + [rust_tokenizer.unk_token]
         __snake_case:						str   =						[14, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(A		)							,				A		)




       def 							UpperCAmelCase__ (  self :    Tuple							,				*A :    Tuple							,				**A :    List[str]		):
         # It's very difficult to mix/test pretokenization with byte-level
         # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
         pass




       def 							UpperCAmelCase__ (  self :    Dict							,				A :    Union[str, Any]=15		):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
             __snake_case:						Optional[Any]   =						self.rust_tokenizer_class.from_pretrained(A							,				**A		)

             # Simple input
             __snake_case:						List[str]   =						"""This is a simple input"""
             __snake_case:						Optional[Any]   =						["""This is a simple input 1""", """This is a simple input 2"""]
             __snake_case:						List[str]   =						("""This is a simple input""", """This is a pair""")
             __snake_case:						Dict   =						[
                 ("""This is a simple input 1""", """This is a simple input 2"""),
                 ("""This is a simple pair 1""", """This is a simple pair 2"""),
             ]

             # Simple input tests
             self.assertRaises(A							,				tokenizer_r.encode							,				A							,				max_length=A							,				padding="""max_length"""		)

             # Simple input
             self.assertRaises(A							,				tokenizer_r.encode_plus							,				A							,				max_length=A							,				padding="""max_length"""		)

             # Simple input
             self.assertRaises(
                 A							,				tokenizer_r.batch_encode_plus							,				A							,				max_length=A							,				padding="""max_length"""							,				)

             # Pair input
             self.assertRaises(A							,				tokenizer_r.encode							,				A							,				max_length=A							,				padding="""max_length"""		)

             # Pair input
             self.assertRaises(A							,				tokenizer_r.encode_plus							,				A							,				max_length=A							,				padding="""max_length"""		)

             # Pair input
             self.assertRaises(
                 A							,				tokenizer_r.batch_encode_plus							,				A							,				max_length=A							,				padding="""max_length"""							,				)




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         __snake_case:						Optional[Any]   =						CodeGenTokenizer.from_pretrained(self.tmpdirname							,				pad_token="""<pad>"""		)

         # Simple input
         __snake_case:						Dict   =						"""This is a simple input"""
         __snake_case:						int   =						["""This is a simple input looooooooong""", """This is a simple input"""]
         __snake_case:						str   =						("""This is a simple input""", """This is a pair""")
         __snake_case:						Any   =						[
             ("""This is a simple input loooooong""", """This is a simple input"""),
             ("""This is a simple pair loooooong""", """This is a simple pair"""),
         ]

         __snake_case:						str   =						tokenizer.pad_token_id

         __snake_case:						List[Any]   =						tokenizer(A							,				padding="""max_length"""							,				max_length=30							,				return_tensors="""np"""		)
         __snake_case:						Union[str, Any]   =						tokenizer(A							,				padding=A							,				truncate=A							,				return_tensors="""np"""		)
         __snake_case:						Union[str, Any]   =						tokenizer(*A							,				padding="""max_length"""							,				max_length=60							,				return_tensors="""np"""		)
         __snake_case:						int   =						tokenizer(A							,				padding=A							,				truncate=A							,				return_tensors="""np"""		)

         # s
         # test single string max_length padding
         self.assertEqual(out_s["""input_ids"""].shape[-1]							,				30		)
         self.assertTrue(pad_token_id in out_s["""input_ids"""]		)
         self.assertTrue(0 in out_s["""attention_mask"""]		)

         # s2
         # test automatic padding
         self.assertEqual(out_sa["""input_ids"""].shape[-1]							,				33		)
         # long slice doesn't have padding
         self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]		)
         self.assertFalse(0 in out_sa["""attention_mask"""][0]		)
         # short slice does have padding
         self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]		)
         self.assertTrue(0 in out_sa["""attention_mask"""][1]		)

         # p
         # test single pair max_length padding
         self.assertEqual(out_p["""input_ids"""].shape[-1]							,				60		)
         self.assertTrue(pad_token_id in out_p["""input_ids"""]		)
         self.assertTrue(0 in out_p["""attention_mask"""]		)

         # p2
         # test automatic padding pair
         self.assertEqual(out_pa["""input_ids"""].shape[-1]							,				52		)
         # long slice pair doesn't have padding
         self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]		)
         self.assertFalse(0 in out_pa["""attention_mask"""][0]		)
         # short slice pair does have padding
         self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]		)
         self.assertTrue(0 in out_pa["""attention_mask"""][1]		)




       def 							UpperCAmelCase__ (  self :    Optional[int]		):
         __snake_case:						Dict   =						"""$$$"""
         __snake_case:						List[str]   =						CodeGenTokenizer.from_pretrained(self.tmpdirname							,				bos_token=A							,				add_bos_token=A		)

         __snake_case:						Any   =						"""This is a simple input"""
         __snake_case:						int   =						["""This is a simple input 1""", """This is a simple input 2"""]

         __snake_case:						Tuple   =						tokenizer.bos_token_id

         __snake_case:						List[str]   =						tokenizer(A		)
         __snake_case:						Optional[Any]   =						tokenizer(A		)

         self.assertEqual(out_s.input_ids[0]							,				A		)
         self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids		)		)

         __snake_case:						Union[str, Any]   =						tokenizer.decode(out_s.input_ids		)
         __snake_case:						int   =						tokenizer.batch_decode(out_sa.input_ids		)

         self.assertEqual(decode_s.split()[0]							,				A		)
         self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa		)		)




       @slow
       def 							UpperCAmelCase__ (  self :    List[str]		):
         __snake_case:						Optional[int]   =						CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""		)

         __snake_case:						List[str]   =						"""\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
         __snake_case:						List[str]   =						"""\nif len_a > len_b:      result = a\nelse:      result = b"""

         __snake_case:						List[str]   =						tokenizer.encode(A		)
         __snake_case:						Union[str, Any]   =						["""^#""", re.escape("""<|endoftext|>"""		), """^'''""", """^\"\"\"""", """\n\n\n"""]
         __snake_case:						str   =						tokenizer.decode(A							,				truncate_before_pattern=A		)
         self.assertEqual(A							,				A		)






       def 							UpperCAmelCase__ (  self :    int		):
         pass













from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


__UpperCAmelCase       :   Union[str, Any]				       =							{}

try:
      if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase       :   str				       =							["BartphoTokenizer"]

if TYPE_CHECKING:
      try:
            if not is_sentencepiece_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_bartpho import BartphoTokenizer

else:
      import sys

      __UpperCAmelCase       :   Optional[Any]				       =							_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






