


import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    GPTNeoXForCausalLM,
			    GPTNeoXForQuestionAnswering,
			    GPTNeoXForSequenceClassification,
			    GPTNeoXForTokenClassification,
			    GPTNeoXModel,
			)

class 							a						:




							"""simple docstring"""


							def __init__(				self			:    Tuple       ,      lowerCamelCase			:    int       ,      lowerCamelCase			:    Optional[Any]=13       ,      lowerCamelCase			:    Optional[Any]=7       ,      lowerCamelCase			:    Union[str, Any]=True       ,      lowerCamelCase			:    Optional[Any]=True       ,      lowerCamelCase			:    Optional[int]=True       ,      lowerCamelCase			:    List[Any]=True       ,      lowerCamelCase			:    List[str]=99       ,      lowerCamelCase			:    str=64       ,      lowerCamelCase			:    Tuple=5       ,      lowerCamelCase			:    str=4       ,      lowerCamelCase			:    Any=37       ,      lowerCamelCase			:    Union[str, Any]="gelu"       ,      lowerCamelCase			:    str=0.1       ,      lowerCamelCase			:    str=0.1       ,      lowerCamelCase			:    Any=512       ,      lowerCamelCase			:    Tuple=16       ,      lowerCamelCase			:    List[str]=2       ,      lowerCamelCase			:    Dict=0.02       ,      lowerCamelCase			:    List[Any]=3       ,      lowerCamelCase			:    Any=4       ,      lowerCamelCase			:    Optional[int]=None       ,      )      ->       int:
														__snake_case						:     Union[str, Any]									=  parent
														__snake_case						:     List[str]									=  batch_size
														__snake_case						:     str									=  seq_length
														__snake_case						:     Tuple									=  is_training
														__snake_case						:     str									=  use_input_mask
														__snake_case						:     Any									=  use_token_type_ids
														__snake_case						:     List[Any]									=  use_labels
														__snake_case						:     str									=  vocab_size
														__snake_case						:     Any									=  hidden_size
														__snake_case						:     Optional[int]									=  num_hidden_layers
														__snake_case						:     Tuple									=  num_attention_heads
														__snake_case						:     Tuple									=  intermediate_size
														__snake_case						:     str									=  hidden_act
														__snake_case						:     List[Any]									=  hidden_dropout_prob
														__snake_case						:     Any									=  attention_probs_dropout_prob
														__snake_case						:     Dict									=  max_position_embeddings
														__snake_case						:     int									=  type_vocab_size
														__snake_case						:     Tuple									=  type_sequence_label_size
														__snake_case						:     Dict									=  initializer_range
														__snake_case						:     Tuple									=  num_labels
														__snake_case						:     str									=  num_choices
														__snake_case						:     Union[str, Any]									=  scope
														__snake_case						:     int									=  vocab_size - 1

							def        __snake_case   (				self			:    Tuple							)      ->       Optional[int]:
														__snake_case						:     List[Any]									=  ids_tensor([self.batch_size, self.seq_length]       ,      self.vocab_size							)

														__snake_case						:     Any									=  None
														if self.use_input_mask:
																					__snake_case						:     Optional[Any]									=  random_attention_mask([self.batch_size, self.seq_length]							)

														__snake_case						:     Dict									=  None
														if self.use_labels:
																					__snake_case						:     Optional[int]									=  ids_tensor([self.batch_size, self.seq_length]       ,      self.num_labels							)

														__snake_case						:     Optional[Any]									=  self.get_config()

														return config, input_ids, input_mask, token_labels

							def        __snake_case   (				self			:    Optional[Any]							)      ->       List[str]:
														return GPTNeoXConfig(
														    vocab_size=self.vocab_size       ,      hidden_size=self.hidden_size       ,      num_hidden_layers=self.num_hidden_layers       ,      num_attention_heads=self.num_attention_heads       ,      intermediate_size=self.intermediate_size       ,      hidden_act=self.hidden_act       ,      hidden_dropout_prob=self.hidden_dropout_prob       ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,      max_position_embeddings=self.max_position_embeddings       ,      type_vocab_size=self.type_vocab_size       ,      is_decoder=lowerCamelCase       ,      initializer_range=self.initializer_range       ,      pad_token_id=self.pad_token_id       ,      )

							def        __snake_case   (				self			:    List[str]							)      ->       str:
														__snake_case					,     __snake_case					,     __snake_case					,     __snake_case						:     int									=  self.prepare_config_and_inputs()

														__snake_case						:     List[Any]									=  True

														return config, input_ids, input_mask, token_labels

							def        __snake_case   (				self			:    Optional[Any]       ,      lowerCamelCase			:    Optional[int]       ,      lowerCamelCase			:    int       ,      lowerCamelCase			:    List[str]							)      ->       Optional[Any]:
														__snake_case						:     Dict									=  GPTNeoXModel(config=lowerCamelCase							)
														model.to(lowerCamelCase							)
														model.eval()
														__snake_case						:     List[Any]									=  model(lowerCamelCase       ,      attention_mask=lowerCamelCase							)
														__snake_case						:     Any									=  model(lowerCamelCase							)
														self.parent.assertEqual(result.last_hidden_state.shape       ,      (self.batch_size, self.seq_length, self.hidden_size)							)

							def        __snake_case   (				self			:    str       ,      lowerCamelCase			:    Tuple       ,      lowerCamelCase			:    Optional[int]       ,      lowerCamelCase			:    Optional[int]							)      ->       int:
														__snake_case						:     Tuple									=  True
														__snake_case						:     Dict									=  GPTNeoXModel(lowerCamelCase							)
														model.to(lowerCamelCase							)
														model.eval()
														__snake_case						:     Union[str, Any]									=  model(lowerCamelCase       ,      attention_mask=lowerCamelCase							)
														self.parent.assertEqual(result.last_hidden_state.shape       ,      (self.batch_size, self.seq_length, self.hidden_size)							)

							def        __snake_case   (				self			:    int       ,      lowerCamelCase			:    Any       ,      lowerCamelCase			:    List[Any]       ,      lowerCamelCase			:    List[str]       ,      lowerCamelCase			:    Optional[Any]							)      ->       Tuple:
														__snake_case						:     str									=  GPTNeoXForCausalLM(config=lowerCamelCase							)
														model.to(lowerCamelCase							)
														model.eval()
														__snake_case						:     Any									=  model(lowerCamelCase       ,      attention_mask=lowerCamelCase       ,      labels=lowerCamelCase							)
														self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.seq_length, self.vocab_size)							)

							def        __snake_case   (				self			:    Optional[int]       ,      lowerCamelCase			:    int       ,      lowerCamelCase			:    Union[str, Any]       ,      lowerCamelCase			:    Any       ,      lowerCamelCase			:    Union[str, Any]							)      ->       Any:
														__snake_case						:     Any									=  self.num_labels
														__snake_case						:     str									=  GPTNeoXForQuestionAnswering(lowerCamelCase							)
														model.to(lowerCamelCase							)
														model.eval()
														__snake_case						:     Optional[Any]									=  model(lowerCamelCase       ,      attention_mask=lowerCamelCase							)
														self.parent.assertEqual(result.start_logits.shape       ,      (self.batch_size, self.seq_length)							)
														self.parent.assertEqual(result.end_logits.shape       ,      (self.batch_size, self.seq_length)							)

							def        __snake_case   (				self			:    Optional[Any]       ,      lowerCamelCase			:    int       ,      lowerCamelCase			:    Tuple       ,      lowerCamelCase			:    Dict       ,      lowerCamelCase			:    Dict							)      ->       Union[str, Any]:
														__snake_case						:     Any									=  self.num_labels
														__snake_case						:     List[Any]									=  GPTNeoXForSequenceClassification(lowerCamelCase							)
														model.to(lowerCamelCase							)
														model.eval()
														__snake_case						:     Union[str, Any]									=  ids_tensor([self.batch_size]       ,      self.type_sequence_label_size							)
														__snake_case						:     Union[str, Any]									=  model(lowerCamelCase       ,      attention_mask=lowerCamelCase       ,      labels=lowerCamelCase							)
														self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.num_labels)							)

							def        __snake_case   (				self			:    Optional[Any]       ,      lowerCamelCase			:    str       ,      lowerCamelCase			:    Any       ,      lowerCamelCase			:    int       ,      lowerCamelCase			:    Any							)      ->       Optional[int]:
														__snake_case						:     Tuple									=  self.num_labels
														__snake_case						:     Dict									=  GPTNeoXForTokenClassification(lowerCamelCase							)
														model.to(lowerCamelCase							)
														model.eval()
														__snake_case						:     List[Any]									=  model(lowerCamelCase       ,      attention_mask=lowerCamelCase       ,      labels=lowerCamelCase							)
														self.parent.assertEqual(result.logits.shape       ,      (self.batch_size, self.seq_length, self.num_labels)							)

							def        __snake_case   (				self			:    Optional[Any]       ,      lowerCamelCase			:    Tuple       ,      lowerCamelCase			:    List[Any]       ,      lowerCamelCase			:    Dict							)      ->       List[Any]:
														__snake_case						:     Tuple									=  True
														__snake_case						:     int									=  GPTNeoXForCausalLM(config=lowerCamelCase							)
														model.to(lowerCamelCase							)
														model.eval()

														# first forward pass
														__snake_case						:     Tuple									=  model(lowerCamelCase       ,      attention_mask=lowerCamelCase       ,      use_cache=lowerCamelCase							)
														__snake_case						:     Any									=  outputs.past_key_values

														# create hypothetical multiple next token and extent to next_input_ids
														__snake_case						:     Dict									=  ids_tensor((self.batch_size, 3)       ,      config.vocab_size							)
														__snake_case						:     Dict									=  ids_tensor((self.batch_size, 3)       ,      vocab_size=2							)

														# append to next input_ids and
														__snake_case						:     Union[str, Any]									=  torch.cat([input_ids, next_tokens]       ,      dim=-1							)
														__snake_case						:     Optional[int]									=  torch.cat([input_mask, next_mask]       ,      dim=-1							)

														__snake_case						:     str									=  model(lowerCamelCase       ,      attention_mask=lowerCamelCase       ,      output_hidden_states=lowerCamelCase							)
														__snake_case						:     Optional[int]									=  output_from_no_past["hidden_states"][0]
														__snake_case						:     Optional[Any]									=  model(
														    lowerCamelCase       ,      attention_mask=lowerCamelCase       ,      past_key_values=lowerCamelCase       ,      output_hidden_states=lowerCamelCase       ,      )["hidden_states"][0]

														# select random slice
														__snake_case						:     List[str]									=  ids_tensor((1,)       ,      output_from_past.shape[-1]							).item()
														__snake_case						:     str									=  output_from_no_past[:, -3:, random_slice_idx].detach()
														__snake_case						:     Tuple									=  output_from_past[:, :, random_slice_idx].detach()

														self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]							)

														# test that outputs are equal for slice
														self.parent.assertTrue(torch.allclose(lowerCamelCase       ,      lowerCamelCase       ,      atol=1E-3							)							)






							def        __snake_case   (				self			:    List[Any]							)      ->       Tuple:
														__snake_case						:     List[Any]									=  self.prepare_config_and_inputs()
														__snake_case					,     __snake_case					,     __snake_case					,     __snake_case						:     Dict									=  config_and_inputs
														__snake_case						:     Union[str, Any]									=  {"input_ids": input_ids, "attention_mask": input_mask}
														return config, inputs_dict

@require_torch
class 							a						(_lowerCAmelCase	,							_lowerCAmelCase	,							_lowerCAmelCase	,							unittest.TestCase ):




							"""simple docstring"""


							__UpperCAmelCase       :				int	    =    (
							    (
							        GPTNeoXModel,
							        GPTNeoXForCausalLM,
							        GPTNeoXForQuestionAnswering,
							        GPTNeoXForSequenceClassification,
							        GPTNeoXForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							__UpperCAmelCase       :				str	    =    (GPTNeoXForCausalLM,) if is_torch_available() else ()
							__UpperCAmelCase       :				Any	    =    (
							    {
							        "feature-extraction": GPTNeoXModel,
							        "question-answering": GPTNeoXForQuestionAnswering,
							        "text-classification": GPTNeoXForSequenceClassification,
							        "text-generation": GPTNeoXForCausalLM,
							        "token-classification": GPTNeoXForTokenClassification,
							        "zero-shot": GPTNeoXForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							__UpperCAmelCase       :				List[Any]	    =    False
							__UpperCAmelCase       :				List[str]	    =    False
							__UpperCAmelCase       :				Dict	    =    False
							__UpperCAmelCase       :				str	    =    False

							def        __snake_case   (				self			:    Optional[int]							)      ->       int:
														__snake_case						:     List[str]									=  GPTNeoXModelTester(self							)
														__snake_case						:     str									=  ConfigTester(self       ,      config_class=lowerCamelCase       ,      hidden_size=64       ,      num_attention_heads=8							)

							def        __snake_case   (				self			:    str							)      ->       Tuple:
														self.config_tester.run_common_tests()

							def        __snake_case   (				self			:    Optional[Any]							)      ->       str:
														__snake_case					,     __snake_case					,     __snake_case					,     __snake_case						:     List[Any]									=  self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model(lowerCamelCase       ,      lowerCamelCase       ,      lowerCamelCase							)

							def        __snake_case   (				self			:    Any							)      ->       List[str]:
														__snake_case					,     __snake_case					,     __snake_case					,     __snake_case						:     Any									=  self.model_tester.prepare_config_and_inputs_for_decoder()
														self.model_tester.create_and_check_model_as_decoder(lowerCamelCase       ,      lowerCamelCase       ,      lowerCamelCase							)

							def        __snake_case   (				self			:    str							)      ->       Optional[int]:
														# This regression test was failing with PyTorch < 1.3
														__snake_case					,     __snake_case					,     __snake_case					,     __snake_case						:     str									=  self.model_tester.prepare_config_and_inputs_for_decoder()

														__snake_case						:     Tuple									=  None

														self.model_tester.create_and_check_model_as_decoder(lowerCamelCase       ,      lowerCamelCase       ,      lowerCamelCase							)

							def        __snake_case   (				self			:    Tuple							)      ->       Optional[Any]:
														__snake_case					,     __snake_case					,     __snake_case					,     __snake_case						:     Optional[int]									=  self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_decoder_model_past_large_inputs(lowerCamelCase       ,      lowerCamelCase       ,      lowerCamelCase							)

							def        __snake_case   (				self			:    str							)      ->       Dict:
														__snake_case						:     str									=  self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_causal_lm(*lowerCamelCase							)

							def        __snake_case   (				self			:    str							)      ->       Dict:
														__snake_case						:     int									=  self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_question_answering(*lowerCamelCase							)

							def        __snake_case   (				self			:    List[Any]							)      ->       Tuple:
														__snake_case						:     Union[str, Any]									=  self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase							)

							def        __snake_case   (				self			:    Optional[Any]							)      ->       Optional[int]:
														__snake_case						:     str									=  self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_token_classification(*lowerCamelCase							)

							@unittest.skip(reason="Feed forward chunking is not implemented"							)
							def        __snake_case   (				self			:    List[Any]							)      ->       Optional[int]:
														pass






							@parameterized.expand([("linear",), ("dynamic",)]							)
							def        __snake_case   (				self			:    str       ,      lowerCamelCase			:    Optional[int]							)      ->       str:
														__snake_case					,     __snake_case						:     int									=  self.model_tester.prepare_config_and_inputs_for_common()
														__snake_case						:     int									=  ids_tensor([1, 10]       ,      config.vocab_size							)
														__snake_case						:     Tuple									=  ids_tensor([1, int(config.max_position_embeddings * 1.5							)]       ,      config.vocab_size							)

														set_seed(42							)  # Fixed seed at init time so the two models get the same random weights
														__snake_case						:     str									=  GPTNeoXModel(lowerCamelCase							)
														original_model.to(lowerCamelCase							)
														original_model.eval()
														__snake_case						:     List[Any]									=  original_model(lowerCamelCase							).last_hidden_state
														__snake_case						:     Tuple									=  original_model(lowerCamelCase							).last_hidden_state

														set_seed(42							)  # Fixed seed at init time so the two models get the same random weights
														__snake_case						:     Union[str, Any]									=  {"type": scaling_type, "factor": 10.0}
														__snake_case						:     List[Any]									=  GPTNeoXModel(lowerCamelCase							)
														scaled_model.to(lowerCamelCase							)
														scaled_model.eval()
														__snake_case						:     List[Any]									=  scaled_model(lowerCamelCase							).last_hidden_state
														__snake_case						:     Any									=  scaled_model(lowerCamelCase							).last_hidden_state

														# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
														# maximum sequence length, so the outputs for the short input should match.
														if scaling_type == "dynamic":
																					self.assertTrue(torch.allclose(lowerCamelCase       ,      lowerCamelCase       ,      atol=1E-5							)							)
														else:
																					self.assertFalse(torch.allclose(lowerCamelCase       ,      lowerCamelCase       ,      atol=1E-5							)							)

														# The output should be different for long inputs
														self.assertFalse(torch.allclose(lowerCamelCase       ,      lowerCamelCase       ,      atol=1E-5							)							)

@require_torch
class 							a						(unittest.TestCase ):




							"""simple docstring"""


							@slow
							def        __snake_case   (				self			:    Union[str, Any]							)      ->       List[Any]:
														__snake_case						:     List[Any]									=  AutoTokenizer.from_pretrained("EleutherAI/pythia-410m-deduped"							)
														for checkpointing in [True, False]:
																					__snake_case						:     Union[str, Any]									=  GPTNeoXForCausalLM.from_pretrained("EleutherAI/pythia-410m-deduped"							)

																					if checkpointing:
																												model.gradient_checkpointing_enable()
																					else:
																												model.gradient_checkpointing_disable()
																					model.to(lowerCamelCase							)

																					__snake_case						:     Dict									=  tokenizer("My favorite food is"       ,      return_tensors="pt"							).to(lowerCamelCase							)
																					# The hub repo. is updated on 2023-04-04, resulting in poor outputs.
																					# See: https://github.com/huggingface/transformers/pull/24193
																					__snake_case						:     Optional[Any]									=  "My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI'm not sure"

																					__snake_case						:     List[Any]									=  model.generate(**lowerCamelCase       ,      do_sample=lowerCamelCase       ,      max_new_tokens=20							)
																					__snake_case						:     List[str]									=  tokenizer.batch_decode(lowerCamelCase							)[0]

																					self.assertEqual(lowerCamelCase       ,      lowerCamelCase							)









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__a   :							Optional[int]						= {"configuration_xlnet": ["XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP", "XLNetConfig"]}

try:
  if not is_sentencepiece_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __a   :							Tuple						= ["XLNetTokenizer"]

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __a   :							List[str]						= ["XLNetTokenizerFast"]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __a   :							Any						= [
      "XLNET_PRETRAINED_MODEL_ARCHIVE_LIST",
      "XLNetForMultipleChoice",
      "XLNetForQuestionAnswering",
      "XLNetForQuestionAnsweringSimple",
      "XLNetForSequenceClassification",
      "XLNetForTokenClassification",
      "XLNetLMHeadModel",
      "XLNetModel",
      "XLNetPreTrainedModel",
      "load_tf_weights_in_xlnet",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __a   :							Dict						= [
      "TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST",
      "TFXLNetForMultipleChoice",
      "TFXLNetForQuestionAnsweringSimple",
      "TFXLNetForSequenceClassification",
      "TFXLNetForTokenClassification",
      "TFXLNetLMHeadModel",
      "TFXLNetMainLayer",
      "TFXLNetModel",
      "TFXLNetPreTrainedModel",
  ]


if TYPE_CHECKING:
  from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig

  try:
    if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_xlnet import XLNetTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_xlnet_fast import XLNetTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_xlnet import (
        XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
        XLNetForMultipleChoice,
        XLNetForQuestionAnswering,
        XLNetForQuestionAnsweringSimple,
        XLNetForSequenceClassification,
        XLNetForTokenClassification,
        XLNetLMHeadModel,
        XLNetModel,
        XLNetPreTrainedModel,
        load_tf_weights_in_xlnet,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_xlnet import (
        TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLNetForMultipleChoice,
        TFXLNetForQuestionAnsweringSimple,
        TFXLNetForSequenceClassification,
        TFXLNetForTokenClassification,
        TFXLNetLMHeadModel,
        TFXLNetMainLayer,
        TFXLNetModel,
        TFXLNetPreTrainedModel,
    )

else:
  import sys

  __a   :							str						= _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




