





"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A											=		logging.get_logger(__name__)

__A											=		{}






class 					_lowerCAmelCase   (     a       ):
       """simple docstring"""




       __magic_name__     :Dict		  =					"""llama"""
       __magic_name__     :List[str]		  =					["""past_key_values"""]






       def __init__(    self    , __UpperCAmelCase=3_2_0_0_0    , __UpperCAmelCase=4_0_9_6    , __UpperCAmelCase=1_1_0_0_8    , __UpperCAmelCase=3_2    , __UpperCAmelCase=3_2    , __UpperCAmelCase=None    , __UpperCAmelCase="silu"    , __UpperCAmelCase=2_0_4_8    , __UpperCAmelCase=0.02    , __UpperCAmelCase=1E-6    , __UpperCAmelCase=True    , __UpperCAmelCase=0    , __UpperCAmelCase=1    , __UpperCAmelCase=2    , __UpperCAmelCase=1    , __UpperCAmelCase=False    , __UpperCAmelCase=None    , **__UpperCAmelCase    , ):



         '''simple docstring'''

         lowerCAmelCase__			:str											=							vocab_size
         lowerCAmelCase__			:List[str]											=							max_position_embeddings
         lowerCAmelCase__			:int											=							hidden_size
         lowerCAmelCase__			:Tuple											=							intermediate_size
         lowerCAmelCase__			:Dict											=							num_hidden_layers
         lowerCAmelCase__			:int											=							num_attention_heads

         # for backward compatibility
         if num_key_value_heads is None:
           lowerCAmelCase__			:Any											=							num_attention_heads

         lowerCAmelCase__			:Any											=							num_key_value_heads
         lowerCAmelCase__			:List[Any]											=							hidden_act
         lowerCAmelCase__			:int											=							initializer_range
         lowerCAmelCase__			:str											=							rms_norm_eps
         lowerCAmelCase__			:Optional[int]											=							pretraining_tp
         lowerCAmelCase__			:str											=							use_cache
         lowerCAmelCase__			:str											=							rope_scaling
         self._rope_scaling_validation()

         super().__init__(
             pad_token_id=__UpperCAmelCase    , bos_token_id=__UpperCAmelCase    , eos_token_id=__UpperCAmelCase    , tie_word_embeddings=__UpperCAmelCase    , **__UpperCAmelCase    , )






       def      snake_case						(    self      ):



         '''simple docstring'''

         if self.rope_scaling is None:
           return

         if not isinstance(self.rope_scaling    , __UpperCAmelCase      ) or len(self.rope_scaling      ) != 2:
           raise ValueError(
               '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
               F"got {self.rope_scaling}"      )
         lowerCAmelCase__			:str											=							self.rope_scaling.get('type'    , __UpperCAmelCase      )
         lowerCAmelCase__			:Dict											=							self.rope_scaling.get('factor'    , __UpperCAmelCase      )
         if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
           raise ValueError(
               F"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"      )
         if rope_scaling_factor is None or not isinstance(__UpperCAmelCase    , __UpperCAmelCase      ) or rope_scaling_factor <= 1.0:
           raise ValueError(F"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"      )







"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


__A											=		{"""configuration_reformer""": ["""REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""", """ReformerConfig"""]}

try:
  if not is_sentencepiece_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __A											=		["""ReformerTokenizer"""]

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __A											=		["""ReformerTokenizerFast"""]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __A											=		[
      """REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """ReformerAttention""",
      """ReformerForMaskedLM""",
      """ReformerForQuestionAnswering""",
      """ReformerForSequenceClassification""",
      """ReformerLayer""",
      """ReformerModel""",
      """ReformerModelWithLMHead""",
      """ReformerPreTrainedModel""",
  ]


if TYPE_CHECKING:
  from .configuration_reformer import REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, ReformerConfig

  try:
    if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_reformer import ReformerTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_reformer_fast import ReformerTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_reformer import (
        REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
        ReformerAttention,
        ReformerForMaskedLM,
        ReformerForQuestionAnswering,
        ReformerForSequenceClassification,
        ReformerLayer,
        ReformerModel,
        ReformerModelWithLMHead,
        ReformerPreTrainedModel,
    )

else:
  import sys

  __A											=		_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
