



import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class SCREAMING_SNAKE_CASE__     (		SCREAMING_SNAKE_CASE__ ,						unittest.TestCase    ):




    '''simple docstring'''






    __lowerCamelCase			: List[str]					=      CodeGenTokenizer
    __lowerCamelCase			: Any					=      CodeGenTokenizerFast
    __lowerCamelCase			: Any					=      True
    __lowerCamelCase			: Any					=      {"add_prefix_space": True}
    __lowerCamelCase			: int					=      False






    def 			_lowerCAmelCase					(       self		):
        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        A :       str          =   [
            """l""",
            """o""",
            """w""",
            """e""",
            """r""",
            """s""",
            """t""",
            """i""",
            """d""",
            """n""",
            """\u0120""",
            """\u0120l""",
            """\u0120n""",
            """\u0120lo""",
            """\u0120low""",
            """er""",
            """\u0120lowest""",
            """\u0120newer""",
            """\u0120wider""",
            """<unk>""",
            """<|endoftext|>""",
        ]
        A :       List[str]          =   dict(zip(lowerCamelCase__,		range(len(lowerCamelCase__		)		)		)		)
        A :       List[Any]          =   ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
        A :       int          =   {"""unk_token""": """<unk>"""}

        A :       Union[str, Any]          =   os.path.join(self.tmpdirname,		VOCAB_FILES_NAMES["""vocab_file"""]		)
        A :       Union[str, Any]          =   os.path.join(self.tmpdirname,		VOCAB_FILES_NAMES["""merges_file"""]		)
        with open(self.vocab_file,		"""w""",		encoding="""utf-8"""		) as fp:
            fp.write(json.dumps(lowerCamelCase__		) + """\n"""		)
        with open(self.merges_file,		"""w""",		encoding="""utf-8"""		) as fp:
            fp.write("""\n""".join(lowerCamelCase__		)		)






    def 			_lowerCAmelCase					(       self,		**lowerCamelCase__		):
        kwargs.update(self.special_tokens_map		)
        return CodeGenTokenizer.from_pretrained(self.tmpdirname,		**lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self,		**lowerCamelCase__		):
        kwargs.update(self.special_tokens_map		)
        return CodeGenTokenizerFast.from_pretrained(self.tmpdirname,		**lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__		):
        A :       Dict          =   """lower newer"""
        A :       Any          =   """lower newer"""
        return input_text, output_text






    def 			_lowerCAmelCase					(       self		):
        A :       int          =   CodeGenTokenizer(self.vocab_file,		self.merges_file,		**self.special_tokens_map		)
        A :       Any          =   """lower newer"""
        A :       Optional[Any]          =   ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
        A :       Optional[Any]          =   tokenizer.tokenize(lowerCamelCase__,		add_prefix_space=lowerCamelCase__		)
        self.assertListEqual(lowerCamelCase__,		lowerCamelCase__		)

        A :       Any          =   tokens + [tokenizer.unk_token]
        A :       Tuple          =   [14, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__		),		lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        if not self.test_rust_tokenizer:
            return

        A :       str          =   self.get_tokenizer()
        A :       str          =   self.get_rust_tokenizer(add_prefix_space=lowerCamelCase__		)

        A :       Any          =   """lower newer"""

        # Testing tokenization
        A :       str          =   tokenizer.tokenize(lowerCamelCase__,		add_prefix_space=lowerCamelCase__		)
        A :       Dict          =   rust_tokenizer.tokenize(lowerCamelCase__		)
        self.assertListEqual(lowerCamelCase__,		lowerCamelCase__		)

        # Testing conversion to ids without special tokens
        A :       Union[str, Any]          =   tokenizer.encode(lowerCamelCase__,		add_special_tokens=lowerCamelCase__,		add_prefix_space=lowerCamelCase__		)
        A :       int          =   rust_tokenizer.encode(lowerCamelCase__,		add_special_tokens=lowerCamelCase__		)
        self.assertListEqual(lowerCamelCase__,		lowerCamelCase__		)

        # Testing conversion to ids with special tokens
        A :       List[Any]          =   self.get_rust_tokenizer(add_prefix_space=lowerCamelCase__		)
        A :       Union[str, Any]          =   tokenizer.encode(lowerCamelCase__,		add_prefix_space=lowerCamelCase__		)
        A :       Any          =   rust_tokenizer.encode(lowerCamelCase__		)
        self.assertListEqual(lowerCamelCase__,		lowerCamelCase__		)

        # Testing the unknown token
        A :       Tuple          =   tokens + [rust_tokenizer.unk_token]
        A :       Any          =   [14, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCamelCase__		),		lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        # It's very difficult to mix/test pretokenization with byte-level
        # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
        pass






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__=15		):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
                A :       Any          =   self.rust_tokenizer_class.from_pretrained(lowerCamelCase__,		**lowerCamelCase__		)

                # Simple input
                A :       Tuple          =   """This is a simple input"""
                A :       List[Any]          =   ["""This is a simple input 1""", """This is a simple input 2"""]
                A :       Union[str, Any]          =   ("""This is a simple input""", """This is a pair""")
                A :       List[Any]          =   [
                    ("""This is a simple input 1""", """This is a simple input 2"""),
                    ("""This is a simple pair 1""", """This is a simple pair 2"""),
                ]

                # Simple input tests
                self.assertRaises(lowerCamelCase__,		tokenizer_r.encode,		lowerCamelCase__,		max_length=lowerCamelCase__,		padding="""max_length"""		)

                # Simple input
                self.assertRaises(lowerCamelCase__,		tokenizer_r.encode_plus,		lowerCamelCase__,		max_length=lowerCamelCase__,		padding="""max_length"""		)

                # Simple input
                self.assertRaises(
                    lowerCamelCase__,		tokenizer_r.batch_encode_plus,		lowerCamelCase__,		max_length=lowerCamelCase__,		padding="""max_length""",		)

                # Pair input
                self.assertRaises(lowerCamelCase__,		tokenizer_r.encode,		lowerCamelCase__,		max_length=lowerCamelCase__,		padding="""max_length"""		)

                # Pair input
                self.assertRaises(lowerCamelCase__,		tokenizer_r.encode_plus,		lowerCamelCase__,		max_length=lowerCamelCase__,		padding="""max_length"""		)

                # Pair input
                self.assertRaises(
                    lowerCamelCase__,		tokenizer_r.batch_encode_plus,		lowerCamelCase__,		max_length=lowerCamelCase__,		padding="""max_length""",		)






    def 			_lowerCAmelCase					(       self		):
        A :       str          =   CodeGenTokenizer.from_pretrained(self.tmpdirname,		pad_token="""<pad>"""		)

        # Simple input
        A :       Optional[Any]          =   """This is a simple input"""
        A :       List[str]          =   ["""This is a simple input looooooooong""", """This is a simple input"""]
        A :       int          =   ("""This is a simple input""", """This is a pair""")
        A :       Union[str, Any]          =   [
            ("""This is a simple input loooooong""", """This is a simple input"""),
            ("""This is a simple pair loooooong""", """This is a simple pair"""),
        ]

        A :       List[Any]          =   tokenizer.pad_token_id

        A :       Dict          =   tokenizer(lowerCamelCase__,		padding="""max_length""",		max_length=30,		return_tensors="""np"""		)
        A :       Optional[int]          =   tokenizer(lowerCamelCase__,		padding=lowerCamelCase__,		truncate=lowerCamelCase__,		return_tensors="""np"""		)
        A :       Union[str, Any]          =   tokenizer(*lowerCamelCase__,		padding="""max_length""",		max_length=60,		return_tensors="""np"""		)
        A :       Dict          =   tokenizer(lowerCamelCase__,		padding=lowerCamelCase__,		truncate=lowerCamelCase__,		return_tensors="""np"""		)

        # s
        # test single string max_length padding
        self.assertEqual(out_s["""input_ids"""].shape[-1],		30		)
        self.assertTrue(pad_token_id in out_s["""input_ids"""]		)
        self.assertTrue(0 in out_s["""attention_mask"""]		)

        # s2
        # test automatic padding
        self.assertEqual(out_sa["""input_ids"""].shape[-1],		33		)
        # long slice doesn't have padding
        self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]		)
        self.assertFalse(0 in out_sa["""attention_mask"""][0]		)
        # short slice does have padding
        self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]		)
        self.assertTrue(0 in out_sa["""attention_mask"""][1]		)

        # p
        # test single pair max_length padding
        self.assertEqual(out_p["""input_ids"""].shape[-1],		60		)
        self.assertTrue(pad_token_id in out_p["""input_ids"""]		)
        self.assertTrue(0 in out_p["""attention_mask"""]		)

        # p2
        # test automatic padding pair
        self.assertEqual(out_pa["""input_ids"""].shape[-1],		52		)
        # long slice pair doesn't have padding
        self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]		)
        self.assertFalse(0 in out_pa["""attention_mask"""][0]		)
        # short slice pair does have padding
        self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]		)
        self.assertTrue(0 in out_pa["""attention_mask"""][1]		)






    def 			_lowerCAmelCase					(       self		):
        A :       str          =   """$$$"""
        A :       Tuple          =   CodeGenTokenizer.from_pretrained(self.tmpdirname,		bos_token=lowerCamelCase__,		add_bos_token=lowerCamelCase__		)

        A :       Union[str, Any]          =   """This is a simple input"""
        A :       Union[str, Any]          =   ["""This is a simple input 1""", """This is a simple input 2"""]

        A :       Dict          =   tokenizer.bos_token_id

        A :       List[Any]          =   tokenizer(lowerCamelCase__		)
        A :       str          =   tokenizer(lowerCamelCase__		)

        self.assertEqual(out_s.input_ids[0],		lowerCamelCase__		)
        self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids		)		)

        A :       Optional[int]          =   tokenizer.decode(out_s.input_ids		)
        A :       Tuple          =   tokenizer.batch_decode(out_sa.input_ids		)

        self.assertEqual(decode_s.split()[0],		lowerCamelCase__		)
        self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa		)		)






    @slow
    def 			_lowerCAmelCase					(       self		):
        A :       Optional[int]          =   CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""		)

        A :       Union[str, Any]          =   """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
        A :       Any          =   """\nif len_a > len_b:      result = a\nelse:      result = b"""

        A :       Optional[int]          =   tokenizer.encode(lowerCamelCase__		)
        A :       Tuple          =   ["""^#""", re.escape("""<|endoftext|>"""		), """^'''""", """^\"\"\"""", """\n\n\n"""]
        A :       Any          =   tokenizer.decode(lowerCamelCase__,		truncate_before_pattern=lowerCamelCase__		)
        self.assertEqual(lowerCamelCase__,		lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self		):
        pass







import unittest

import numpy as np

from transformers import AlbertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
     import jax.numpy as jnp

     from transformers.models.albert.modeling_flax_albert import (
         FlaxAlbertForMaskedLM,
         FlaxAlbertForMultipleChoice,
         FlaxAlbertForPreTraining,
         FlaxAlbertForQuestionAnswering,
         FlaxAlbertForSequenceClassification,
         FlaxAlbertForTokenClassification,
         FlaxAlbertModel,
     )






class SCREAMING_SNAKE_CASE__     (		unittest.TestCase    ):




    '''simple docstring'''






    def __init__(       self,		lowerCamelCase__,		lowerCamelCase__=13,		lowerCamelCase__=7,		lowerCamelCase__=True,		lowerCamelCase__=True,		lowerCamelCase__=True,		lowerCamelCase__=True,		lowerCamelCase__=99,		lowerCamelCase__=32,		lowerCamelCase__=5,		lowerCamelCase__=4,		lowerCamelCase__=37,		lowerCamelCase__="gelu",		lowerCamelCase__=0.1,		lowerCamelCase__=0.1,		lowerCamelCase__=512,		lowerCamelCase__=16,		lowerCamelCase__=2,		lowerCamelCase__=0.02,		lowerCamelCase__=4,		):
        A :       List[str]          =   parent
        A :       Optional[int]          =   batch_size
        A :       Union[str, Any]          =   seq_length
        A :       Any          =   is_training
        A :       List[str]          =   use_attention_mask
        A :       Union[str, Any]          =   use_token_type_ids
        A :       Any          =   use_labels
        A :       str          =   vocab_size
        A :       Union[str, Any]          =   hidden_size
        A :       str          =   num_hidden_layers
        A :       List[Any]          =   num_attention_heads
        A :       Optional[int]          =   intermediate_size
        A :       Optional[Any]          =   hidden_act
        A :       Dict          =   hidden_dropout_prob
        A :       List[Any]          =   attention_probs_dropout_prob
        A :       Optional[int]          =   max_position_embeddings
        A :       int          =   type_vocab_size
        A :       str          =   type_sequence_label_size
        A :       List[Any]          =   initializer_range
        A :       str          =   num_choices






    def 			_lowerCAmelCase					(       self		):
        A :       Optional[int]          =   ids_tensor([self.batch_size, self.seq_length],		self.vocab_size		)

        A :       Union[str, Any]          =   None
        if self.use_attention_mask:
            A :       Optional[int]          =   random_attention_mask([self.batch_size, self.seq_length]		)

        A :       int          =   None
        if self.use_token_type_ids:
            A :       List[Any]          =   ids_tensor([self.batch_size, self.seq_length],		self.type_vocab_size		)

        A :       Optional[int]          =   AlbertConfig(
            vocab_size=self.vocab_size,		hidden_size=self.hidden_size,		num_hidden_layers=self.num_hidden_layers,		num_attention_heads=self.num_attention_heads,		intermediate_size=self.intermediate_size,		hidden_act=self.hidden_act,		hidden_dropout_prob=self.hidden_dropout_prob,		attention_probs_dropout_prob=self.attention_probs_dropout_prob,		max_position_embeddings=self.max_position_embeddings,		type_vocab_size=self.type_vocab_size,		is_decoder=lowerCamelCase__,		initializer_range=self.initializer_range,		)

        return config, input_ids, token_type_ids, attention_mask






    def 			_lowerCAmelCase					(       self		):
        A :       Dict          =   self.prepare_config_and_inputs()
        A     ,							A     ,							A     ,							A :       str          =   config_and_inputs
        A :       Union[str, Any]          =   {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
        return config, inputs_dict






@require_flax
class SCREAMING_SNAKE_CASE__     (		SCREAMING_SNAKE_CASE__ ,						unittest.TestCase    ):




    '''simple docstring'''






    __lowerCamelCase			: Any					=      (
        (
            FlaxAlbertModel,
            FlaxAlbertForPreTraining,
            FlaxAlbertForMaskedLM,
            FlaxAlbertForMultipleChoice,
            FlaxAlbertForQuestionAnswering,
            FlaxAlbertForSequenceClassification,
            FlaxAlbertForTokenClassification,
            FlaxAlbertForQuestionAnswering,
        )
        if is_flax_available()
        else ()
    )






    def 			_lowerCAmelCase					(       self		):
        A :       Dict          =   FlaxAlbertModelTester(self		)






    @slow
    def 			_lowerCAmelCase					(       self		):
        for model_class_name in self.all_model_classes:
            A :       Dict          =   model_class_name.from_pretrained("""albert-base-v2"""		)
            A :       Optional[int]          =   model(np.ones((1, 1)		)		)
            self.assertIsNotNone(lowerCamelCase__		)






@require_flax
class SCREAMING_SNAKE_CASE__     (		unittest.TestCase    ):




    '''simple docstring'''






    @slow
    def 			_lowerCAmelCase					(       self		):
        A :       Dict          =   FlaxAlbertModel.from_pretrained("""albert-base-v2"""		)
        A :       List[str]          =   np.array([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]		)
        A :       str          =   np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]		)
        A :       Optional[int]          =   model(lowerCamelCase__,		attention_mask=lowerCamelCase__		)[0]
        A :       str          =   (1, 11, 768)
        self.assertEqual(output.shape,		lowerCamelCase__		)
        A :       Optional[int]          =   np.array(
            [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]		)

        self.assertTrue(jnp.allclose(output[:, 1:4, 1:4],		lowerCamelCase__,		atol=1e-4		)		)


