






import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
   from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
   import torch_xla.core.xla_model as xm


def 	lowerCamelCase__      (  lowercase					):

  """simple docstring"""






  if is_torch_version("<"    ,  "2.0.0"					) or not hasattr(lowercase    ,  "_dynamo"					):
    return False
  return isinstance(lowercase    ,  torch._dynamo.eval_frame.OptimizedModule					)


def 	lowerCamelCase__      (  lowercase    ,  lowercase = True					):

  """simple docstring"""






  SCREAMING_SNAKE_CASE							:   List[str]						    =     (torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

  SCREAMING_SNAKE_CASE							:   Optional[int]						    =     is_compiled_module(lowercase					)
  if is_compiled:
    SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     model
    SCREAMING_SNAKE_CASE							:   int						    =     model._orig_mod

  if is_deepspeed_available():
    options += (DeepSpeedEngine,)

  while isinstance(lowercase    ,  lowercase					):
    SCREAMING_SNAKE_CASE							:   List[str]						    =     model.module

  if not keep_fpaa_wrapper:
    SCREAMING_SNAKE_CASE							:   Tuple						    =     getattr(lowercase    ,  "forward"					)
    SCREAMING_SNAKE_CASE							:   Any						    =     model.__dict__.pop("_original_forward"    ,  lowercase					)
    if original_forward is not None:
      while hasattr(lowercase    ,  "__wrapped__"					):
        SCREAMING_SNAKE_CASE							:   Any						    =     forward.__wrapped__
        if forward == original_forward:
          break
      SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     forward
    if getattr(lowercase    ,  "_converted_to_transformer_engine"    ,  lowercase					):
      convert_model(lowercase    ,  to_transformer_engine=lowercase					)

  if is_compiled:
    SCREAMING_SNAKE_CASE							:   str						    =     model
    SCREAMING_SNAKE_CASE							:   Optional[Any]						    =     compiled_model

  return model


def 	lowerCamelCase__      (  ):

  """simple docstring"""






  PartialState().wait_for_everyone()


def 	lowerCamelCase__      (  lowercase    ,  lowercase					):

  """simple docstring"""






  if PartialState().distributed_type == DistributedType.TPU:
    xm.save(lowercase    ,  lowercase					)
  elif PartialState().local_process_index == 0:
    torch.save(lowercase    ,  lowercase					)


@contextmanager
def 	lowerCamelCase__      (  **lowercase					):

  """simple docstring"""






  for key, value in kwargs.items():
    SCREAMING_SNAKE_CASE							:   str						    =     str(lowercase					)

  yield

  for key in kwargs:
    if key.upper() in os.environ:
      del os.environ[key.upper()]


def 	lowerCamelCase__      (  lowercase					):

  """simple docstring"""






  if not hasattr(lowercase    ,  "__qualname__"					) and not hasattr(lowercase    ,  "__name__"					):
    SCREAMING_SNAKE_CASE							:   int						    =     getattr(lowercase    ,  "__class__"    ,  lowercase					)
  if hasattr(lowercase    ,  "__qualname__"					):
    return obj.__qualname__
  if hasattr(lowercase    ,  "__name__"					):
    return obj.__name__
  return str(lowercase					)


def 	lowerCamelCase__      (  lowercase    ,  lowercase					):

  """simple docstring"""






  for key, value in source.items():
    if isinstance(lowercase    ,  lowercase					):
      SCREAMING_SNAKE_CASE							:   List[str]						    =     destination.setdefault(lowercase    ,  {}					)
      merge_dicts(lowercase    ,  lowercase					)
    else:
      SCREAMING_SNAKE_CASE							:   Any						    =     value

  return destination


def 	lowerCamelCase__      (  lowercase = None					):

  """simple docstring"""






  if port is None:
    SCREAMING_SNAKE_CASE							:   str						    =     29500
  with socket.socket(socket.AF_INET    ,  socket.SOCK_STREAM					) as s:
    return s.connect_ex(("localhost", port)					) == 0










from __future__ import annotations

from typing import Dict

from ...configuration_utils import PretrainedConfig


SCREAMING_SNAKE_CASE__					:     List[Any]       =     {
    'susnato/ernie-m-base_pytorch': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/config.json',
    'susnato/ernie-m-large_pytorch': 'https://huggingface.co/susnato/ernie-m-large_pytorch/blob/main/config.json',
}


class 		_SCREAMING_SNAKE_CASE     (						A     ):
     __SCREAMING_SNAKE_CASE     					=    '''ernie_m'''
     __SCREAMING_SNAKE_CASE     					=    {"dropout": "classifier_dropout", "num_classes": "num_labels"}


     def __init__(  self							,					A_ = 25_00_02							,					A_ = 7_68							,					A_ = 12							,					A_ = 12							,					A_ = 30_72							,					A_ = "gelu"							,					A_ = 0.1							,					A_ = 0.1							,					A_ = 5_14							,					A_ = 0.0_2							,					A_ = 1							,					A_ = 1e-05							,					A_=None							,					A_=False							,					A_=0.0							,					**A_							,					):
       super().__init__(pad_token_id=A_							,					**A_    )
       _UpperCAmelCase		:							Tuple							=     vocab_size
       _UpperCAmelCase		:							List[str]							=     hidden_size
       _UpperCAmelCase		:							Optional[int]							=     num_hidden_layers
       _UpperCAmelCase		:							Tuple							=     num_attention_heads
       _UpperCAmelCase		:							Dict							=     intermediate_size
       _UpperCAmelCase		:							List[Any]							=     hidden_act
       _UpperCAmelCase		:							Any							=     hidden_dropout_prob
       _UpperCAmelCase		:							Any							=     attention_probs_dropout_prob
       _UpperCAmelCase		:							Tuple							=     max_position_embeddings
       _UpperCAmelCase		:							Dict							=     initializer_range
       _UpperCAmelCase		:							Optional[Any]							=     layer_norm_eps
       _UpperCAmelCase		:							str							=     classifier_dropout
       _UpperCAmelCase		:							Any							=     is_decoder
       _UpperCAmelCase		:							Union[str, Any]							=     act_dropout





