




import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel







class __snake_case			:







   def __init__(      self		:						Any					,						_snake_case		:						int					,						_snake_case		:						List[Any]=13					,						_snake_case		:						int=7					,						_snake_case		:						List[str]=True					,						_snake_case		:						int=True					,						_snake_case		:						Dict=True					,						_snake_case		:						Union[str, Any]=True					,						_snake_case		:						Union[str, Any]=99					,						_snake_case		:						Dict=32					,						_snake_case		:						Dict=5					,						_snake_case		:						str=4					,						_snake_case		:						Optional[int]=4					,						_snake_case		:						Optional[Any]="gelu"					,						_snake_case		:						List[Any]=0.0					,						_snake_case		:						Any=0.1					,						_snake_case		:						Optional[Any]=True					,						_snake_case		:						int=512					,						_snake_case		:						Tuple=16					,						_snake_case		:						str=2					,						_snake_case		:						Any=0.0_2					,						_snake_case		:						Any=3					,						_snake_case		:						Any=4					,						_snake_case		:						List[str]=None					,						):

        """simple docstring"""


        UpperCAmelCase_          =							parent
        UpperCAmelCase_          =							batch_size
        UpperCAmelCase_          =							seq_length
        UpperCAmelCase_          =							is_training
        UpperCAmelCase_          =							use_input_mask
        UpperCAmelCase_          =							use_token_type_ids
        UpperCAmelCase_          =							use_labels
        UpperCAmelCase_          =							vocab_size
        UpperCAmelCase_          =							hidden_size
        UpperCAmelCase_          =							num_hidden_layers
        UpperCAmelCase_          =							num_attention_heads
        UpperCAmelCase_          =							intermediate_multiple_size
        UpperCAmelCase_          =							hidden_act
        UpperCAmelCase_          =							hidden_dropout
        UpperCAmelCase_          =							attention_dropout
        UpperCAmelCase_          =							weight_tying
        UpperCAmelCase_          =							max_position_embeddings
        UpperCAmelCase_          =							type_vocab_size
        UpperCAmelCase_          =							type_sequence_label_size
        UpperCAmelCase_          =							initializer_range
        UpperCAmelCase_          =							num_labels
        UpperCAmelCase_          =							num_choices
        UpperCAmelCase_          =							scope







   def lowerCamelCase						(      self		:						Any):

        """simple docstring"""


        UpperCAmelCase_          =							ids_tensor([self.batch_size, self.seq_length]					,						self.vocab_size)

        UpperCAmelCase_          =							None
        if self.use_input_mask:
             UpperCAmelCase_          =							random_attention_mask([self.batch_size, self.seq_length])

        UpperCAmelCase_          =							None
        if self.use_labels:
             UpperCAmelCase_          =							ids_tensor([self.batch_size, self.seq_length]					,						self.num_labels)

        UpperCAmelCase_          =							self.get_config()

        return config, input_ids, input_mask, token_labels







   def lowerCamelCase						(      self		:						int):

        """simple docstring"""


        return GPTNeoXJapaneseConfig(
            vocab_size=self.vocab_size					,						hidden_size=self.hidden_size					,						num_hidden_layers=self.num_hidden_layers					,						num_attention_heads=self.num_attention_heads					,						intermediate_multiple_size=self.intermediate_multiple_size					,						hidden_act=self.hidden_act					,						hidden_dropout=self.hidden_dropout					,						attention_dropout=self.attention_dropout					,						weight_tying=self.weight_tying					,						max_position_embeddings=self.max_position_embeddings					,						type_vocab_size=self.type_vocab_size					,						is_decoder=_snake_case					,						initializer_range=self.initializer_range					,						)







   def lowerCamelCase						(      self		:						List[Any]):

        """simple docstring"""


        UpperCAmelCase_          =							self.prepare_config_and_inputs()

        UpperCAmelCase_          =							True

        return config, input_ids, input_mask, token_labels







   def lowerCamelCase						(      self		:						Optional[int]					,						_snake_case		:						Tuple					,						_snake_case		:						Optional[int]					,						_snake_case		:						Tuple):

        """simple docstring"""


        UpperCAmelCase_          =							GPTNeoXJapaneseModel(config=_snake_case)
        model.to(_snake_case)
        model.eval()
        UpperCAmelCase_          =							model(_snake_case					,						attention_mask=_snake_case)
        UpperCAmelCase_          =							model(_snake_case)
        self.parent.assertEqual(result.last_hidden_state.shape					,						(self.batch_size, self.seq_length, self.hidden_size))







   def lowerCamelCase						(      self		:						Tuple					,						_snake_case		:						Tuple					,						_snake_case		:						List[str]					,						_snake_case		:						Optional[Any]):

        """simple docstring"""


        UpperCAmelCase_          =							True
        UpperCAmelCase_          =							GPTNeoXJapaneseModel(_snake_case)
        model.to(_snake_case)
        model.eval()
        UpperCAmelCase_          =							model(_snake_case					,						attention_mask=_snake_case)
        self.parent.assertEqual(result.last_hidden_state.shape					,						(self.batch_size, self.seq_length, self.hidden_size))







   def lowerCamelCase						(      self		:						Union[str, Any]					,						_snake_case		:						Any					,						_snake_case		:						Optional[Any]					,						_snake_case		:						Dict					,						_snake_case		:						int):

        """simple docstring"""


        UpperCAmelCase_          =							GPTNeoXJapaneseForCausalLM(config=_snake_case)
        model.to(_snake_case)
        model.eval()
        UpperCAmelCase_          =							model(_snake_case					,						attention_mask=_snake_case					,						labels=_snake_case)
        self.parent.assertEqual(result.logits.shape					,						(self.batch_size, self.seq_length, self.vocab_size))







   def lowerCamelCase						(      self		:						Tuple					,						_snake_case		:						List[Any]					,						_snake_case		:						Union[str, Any]					,						_snake_case		:						Union[str, Any]):

        """simple docstring"""


        UpperCAmelCase_          =							True
        UpperCAmelCase_          =							GPTNeoXJapaneseForCausalLM(config=_snake_case)
        model.to(_snake_case)
        model.eval()

        # first forward pass
        UpperCAmelCase_          =							model(_snake_case					,						attention_mask=_snake_case					,						use_cache=_snake_case)
        UpperCAmelCase_          =							outputs.past_key_values

        # create hypothetical multiple next token and extent to next_input_ids
        UpperCAmelCase_          =							ids_tensor((self.batch_size, 3)					,						config.vocab_size)
        UpperCAmelCase_          =							ids_tensor((self.batch_size, 3)					,						vocab_size=2)

        # append to next input_ids and
        UpperCAmelCase_          =							torch.cat([input_ids, next_tokens]					,						dim=-1)
        UpperCAmelCase_          =							torch.cat([input_mask, next_mask]					,						dim=-1)

        UpperCAmelCase_          =							model(_snake_case					,						attention_mask=_snake_case					,						output_hidden_states=_snake_case)
        UpperCAmelCase_          =							output_from_no_past['''hidden_states'''][0]
        UpperCAmelCase_          =							model(
            _snake_case					,						attention_mask=_snake_case					,						past_key_values=_snake_case					,						output_hidden_states=_snake_case					,						)['''hidden_states'''][0]

        # select random slice
        UpperCAmelCase_          =							ids_tensor((1,)					,						output_from_past.shape[-1]).item()
        UpperCAmelCase_          =							output_from_no_past[:, -3:, random_slice_idx].detach()
        UpperCAmelCase_          =							output_from_past[:, :, random_slice_idx].detach()

        self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

        # test that outputs are equal for slice
        self.parent.assertTrue(torch.allclose(_snake_case					,						_snake_case					,						atol=1e-3))







   def lowerCamelCase						(      self		:						str):

        """simple docstring"""


        UpperCAmelCase_          =							self.prepare_config_and_inputs()
        UpperCAmelCase_          =							config_and_inputs
        UpperCAmelCase_          =							{'''input_ids''': input_ids, '''attention_mask''': input_mask}
        return config, inputs_dict







@require_torch
class __snake_case			(			a ,  a ,  unittest.TestCase		):
   UpperCAmelCase__  :    List[Any]          =  (GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
   UpperCAmelCase__  :    List[str]          =  (GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
   UpperCAmelCase__  :    List[str]          =  (
       {'''feature-extraction''': GPTNeoXJapaneseModel, '''text-generation''': GPTNeoXJapaneseForCausalLM}
       if is_torch_available()
       else {}
   )
   UpperCAmelCase__  :    Optional[Any]          =  False
   UpperCAmelCase__  :    Union[str, Any]          =  False
   UpperCAmelCase__  :    str          =  False
   UpperCAmelCase__  :    Dict          =  False







   def lowerCamelCase						(      self		:						Dict):

        """simple docstring"""


        UpperCAmelCase_          =							GPTNeoXJapaneseModelTester(self)
        UpperCAmelCase_          =							ConfigTester(self					,						config_class=_snake_case					,						hidden_size=37)







   def lowerCamelCase						(      self		:						Dict):

        """simple docstring"""


        self.config_tester.run_common_tests()







   def lowerCamelCase						(      self		:						Any):

        """simple docstring"""


        UpperCAmelCase_          =							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(_snake_case					,						_snake_case					,						_snake_case)







   def lowerCamelCase						(      self		:						Union[str, Any]):

        """simple docstring"""


        UpperCAmelCase_          =							self.model_tester.prepare_config_and_inputs_for_decoder()
        self.model_tester.create_and_check_model_as_decoder(_snake_case					,						_snake_case					,						_snake_case)







   def lowerCamelCase						(      self		:						Optional[Any]):

        """simple docstring"""


        UpperCAmelCase_          =							self.model_tester.prepare_config_and_inputs_for_decoder()

        UpperCAmelCase_          =							None

        self.model_tester.create_and_check_model_as_decoder(_snake_case					,						_snake_case					,						_snake_case)







   def lowerCamelCase						(      self		:						Any):

        """simple docstring"""


        UpperCAmelCase_          =							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_decoder_model_past_large_inputs(_snake_case					,						_snake_case					,						_snake_case)







   def lowerCamelCase						(      self		:						Optional[Any]):

        """simple docstring"""


        UpperCAmelCase_          =							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_causal_lm(*_snake_case)







   @slow
   def lowerCamelCase						(      self		:						Optional[Any]):

        """simple docstring"""


        UpperCAmelCase_          =							'''abeja/gpt-neox-japanese-2.7b'''

        UpperCAmelCase_          =							['''データサイエンティストとは、''', '''100年後に必要とされる会社は、''', '''フルリモートの環境で働くために必要なことは、''', '''国境の長いトンネルを抜けると''', '''美味しい日本食といえば、''']

        UpperCAmelCase_          =							[
            '''データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。''',
            '''100年後に必要とされる会社は、「人」が中心の会社です。''',
            '''フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。''',
            '''国境の長いトンネルを抜けると、そこは雪国だった。''',
            '''美味しい日本食といえば、やっぱりお寿司ですよね。''',
        ]

        UpperCAmelCase_          =							GPTNeoXJapaneseTokenizer.from_pretrained(_snake_case)
        UpperCAmelCase_          =							GPTNeoXJapaneseForCausalLM.from_pretrained(_snake_case)

        UpperCAmelCase_          =							[]
        for prompt in prompts:
             UpperCAmelCase_          =							tokenizer(_snake_case					,						return_tensors='''pt''').input_ids
             UpperCAmelCase_          =							model.generate(_snake_case					,						max_length=50)
             UpperCAmelCase_          =							tokenizer.batch_decode(_snake_case					,						skip_special_tokens=_snake_case)
             predicted_outputs += generated_string
        self.assertListEqual(_snake_case					,						_snake_case)











"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCAmelCase    :List[str]     =					logging.get_logger(__name__)

_lowerCAmelCase    :Any     =					{
    'tiiuae/falcon-40b': 'https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json',
    'tiiuae/falcon-7b': 'https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json',
}
class 							_UpperCAmelCase							( a      ):





     '''simple docstring'''



     a__									='''falcon'''
     a__									=['''past_key_values''']





     def __init__(					self       ,	A=6_5_0_2_4       ,	A=4_5_4_4       ,	A=3_2       ,	A=7_1       ,	A=1E-5       ,	A=0.02       ,	A=True       ,	A=0.0       ,	A=0.0       ,	A=None       ,	A=False       ,	A=False       ,	A=True       ,	A=True       ,	A=False       ,	A=1_1       ,	A=1_1       ,	**A       ,	)							->					Any:
         _UpperCAmelCase							:  int											=    vocab_size
         # Backward compatibility with n_embed kwarg
         _UpperCAmelCase							:  Optional[Any]											=    kwargs.pop('''n_embed'''       ,	A		)
         _UpperCAmelCase							:  int											=    hidden_size if n_embed is None else n_embed
         _UpperCAmelCase							:  List[str]											=    num_hidden_layers
         _UpperCAmelCase							:  Tuple											=    num_attention_heads
         _UpperCAmelCase							:  Optional[int]											=    layer_norm_epsilon
         _UpperCAmelCase							:  Tuple											=    initializer_range
         _UpperCAmelCase							:  Optional[int]											=    use_cache
         _UpperCAmelCase							:  Any											=    hidden_dropout
         _UpperCAmelCase							:  Dict											=    attention_dropout

         _UpperCAmelCase							:  Any											=    bos_token_id
         _UpperCAmelCase							:  List[Any]											=    eos_token_id
         _UpperCAmelCase							:  Tuple											=    num_attention_heads if num_kv_heads is None else num_kv_heads
         _UpperCAmelCase							:  Dict											=    alibi
         _UpperCAmelCase							:  Optional[int]											=    new_decoder_architecture
         _UpperCAmelCase							:  str											=    multi_query  # Ignored when new_decoder_architecture is True
         _UpperCAmelCase							:  Optional[int]											=    parallel_attn
         _UpperCAmelCase							:  Optional[int]											=    bias

         super().__init__(bos_token_id=A       ,	eos_token_id=A       ,	**A		)





     @property
     def  __lowerCAmelCase	(					self		)							->					List[str]:
         return self.hidden_size // self.num_attention_heads





     @property
     def  __lowerCAmelCase	(					self		)							->					List[Any]:
         return not self.alibi


