'''simple docstring'''

from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


__snake_case							: int			=     logging.get_logger(__name__)

__snake_case							: List[str]			=     {
    '''Helsinki-NLP/opus-mt-en-de''': '''https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/config.json''',
    # See all Marian models at https://huggingface.co/models?filter=marian
}
class 				lowercase_     (  _a    ):
    a_   							= """marian"""
    a_   							= ["""past_key_values"""]
    a_   							= {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}







    def __init__(						self							,  UpperCamelCase__=5_8_1_0_1							,  UpperCamelCase__=None							,  UpperCamelCase__=1_0_2_4							,  UpperCamelCase__=1_2							,  UpperCamelCase__=4_0_9_6							,  UpperCamelCase__=1_6							,  UpperCamelCase__=1_2							,  UpperCamelCase__=4_0_9_6							,  UpperCamelCase__=1_6							,  UpperCamelCase__=0.0							,  UpperCamelCase__=0.0							,  UpperCamelCase__=True							,  UpperCamelCase__=True							,  UpperCamelCase__="gelu"							,  UpperCamelCase__=1_0_2_4							,  UpperCamelCase__=0.1							,  UpperCamelCase__=0.0							,  UpperCamelCase__=0.0							,  UpperCamelCase__=0.02							,  UpperCamelCase__=5_8_1_0_0							,  UpperCamelCase__=False							,  UpperCamelCase__=5_8_1_0_0							,  UpperCamelCase__=0							,  UpperCamelCase__=0							,  UpperCamelCase__=True							,  **UpperCamelCase__							,  )					->							int:





     """simple docstring"""

     UpperCAmelCase_												=       vocab_size
     UpperCAmelCase_												=       decoder_vocab_size or vocab_size
     UpperCAmelCase_												=       max_position_embeddings
     UpperCAmelCase_												=       d_model
     UpperCAmelCase_												=       encoder_ffn_dim
     UpperCAmelCase_												=       encoder_layers
     UpperCAmelCase_												=       encoder_attention_heads
     UpperCAmelCase_												=       decoder_ffn_dim
     UpperCAmelCase_												=       decoder_layers
     UpperCAmelCase_												=       decoder_attention_heads
     UpperCAmelCase_												=       dropout
     UpperCAmelCase_												=       attention_dropout
     UpperCAmelCase_												=       activation_dropout
     UpperCAmelCase_												=       activation_function
     UpperCAmelCase_												=       init_std
     UpperCAmelCase_												=       encoder_layerdrop
     UpperCAmelCase_												=       decoder_layerdrop
     UpperCAmelCase_												=       use_cache
     UpperCAmelCase_												=       encoder_layers
     UpperCAmelCase_												=       scale_embedding  # scale factor will be sqrt(d_model) if True
     UpperCAmelCase_												=       share_encoder_decoder_embeddings
     super().__init__(
         pad_token_id=__lowerCAmelCase							,  eos_token_id=__lowerCAmelCase							,  is_encoder_decoder=__lowerCAmelCase							,  decoder_start_token_id=__lowerCAmelCase							,  forced_eos_token_id=__lowerCAmelCase							,  **__lowerCAmelCase							,  )


class 				lowercase_     (  _a    ):







    @property
    # Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.inputs
    def 	lowerCamelCase_     (						self					)					->							List[Any]:





     """simple docstring"""

     if self.task in ["default", "seq2seq-lm"]:
      UpperCAmelCase_												=       OrderedDict(
          [
              ("input_ids", {0: "batch", 1: "encoder_sequence"}),
              ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
          ]					)

      if self.use_past:
       UpperCAmelCase_												=       {0: "batch"}
       UpperCAmelCase_												=       {0: "batch", 1: "past_decoder_sequence + sequence"}
      else:
       UpperCAmelCase_												=       {0: "batch", 1: "decoder_sequence"}
       UpperCAmelCase_												=       {0: "batch", 1: "decoder_sequence"}

      if self.use_past:
       self.fill_with_past_key_values_(__lowerCAmelCase							,  direction="inputs"					)
     elif self.task == "causal-lm":
      # TODO: figure this case out.
      UpperCAmelCase_												=       OrderedDict(
          [
              ("input_ids", {0: "batch", 1: "encoder_sequence"}),
              ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
          ]					)
      if self.use_past:
       UpperCAmelCase_							,							UpperCAmelCase_												=       self.num_layers
       for i in range(__lowerCAmelCase					):
        UpperCAmelCase_												=       {0: "batch", 2: "past_sequence + sequence"}
        UpperCAmelCase_												=       {0: "batch", 2: "past_sequence + sequence"}
     else:
      UpperCAmelCase_												=       OrderedDict(
          [
              ("input_ids", {0: "batch", 1: "encoder_sequence"}),
              ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
              ("decoder_input_ids", {0: "batch", 1: "decoder_sequence"}),
              ("decoder_attention_mask", {0: "batch", 1: "decoder_sequence"}),
          ]					)

     return common_inputs







    @property
    # Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
    def 	lowerCamelCase_     (						self					)					->							Tuple:





     """simple docstring"""

     if self.task in ["default", "seq2seq-lm"]:
      UpperCAmelCase_												=       super().outputs
     else:
      UpperCAmelCase_												=       super(__lowerCAmelCase							,  self					).outputs
      if self.use_past:
       UpperCAmelCase_							,							UpperCAmelCase_												=       self.num_layers
       for i in range(__lowerCAmelCase					):
        UpperCAmelCase_												=       {0: "batch", 2: "past_sequence + sequence"}
        UpperCAmelCase_												=       {0: "batch", 2: "past_sequence + sequence"}
     return common_outputs







    def 	lowerCamelCase_     (						self							,  UpperCamelCase__							,  UpperCamelCase__ = -1							,  UpperCamelCase__ = -1							,  UpperCamelCase__ = False							,  UpperCamelCase__ = None							,  )					->							List[str]:





     """simple docstring"""

     UpperCAmelCase_												=       self._generate_dummy_inputs_for_encoder_and_decoder(
         __lowerCAmelCase							,  __lowerCAmelCase							,  __lowerCAmelCase							,  __lowerCAmelCase							,  __lowerCAmelCase					)

     # Generate decoder inputs
     UpperCAmelCase_												=       seq_length if not self.use_past else 1
     UpperCAmelCase_												=       self._generate_dummy_inputs_for_encoder_and_decoder(
         __lowerCAmelCase							,  __lowerCAmelCase							,  __lowerCAmelCase							,  __lowerCAmelCase							,  __lowerCAmelCase					)
     UpperCAmelCase_												=       {F"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
     UpperCAmelCase_												=       dict(**__lowerCAmelCase							,  **__lowerCAmelCase					)

     if self.use_past:
      if not is_torch_available():
       raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."					)
      else:
       import torch
      UpperCAmelCase_							,							UpperCAmelCase_												=       common_inputs["input_ids"].shape
      UpperCAmelCase_												=       common_inputs["decoder_input_ids"].shape[1]
      UpperCAmelCase_							,							UpperCAmelCase_												=       self.num_attention_heads
      UpperCAmelCase_												=       (
          batch,
          num_encoder_attention_heads,
          encoder_seq_length,
          self._config.hidden_size // num_encoder_attention_heads,
      )
      UpperCAmelCase_												=       decoder_seq_length + 3
      UpperCAmelCase_												=       (
          batch,
          num_decoder_attention_heads,
          decoder_past_length,
          self._config.hidden_size // num_decoder_attention_heads,
      )

      UpperCAmelCase_												=       torch.cat(
          [common_inputs["decoder_attention_mask"], torch.ones(__lowerCAmelCase							,  __lowerCAmelCase					)]							,  dim=1					)

      UpperCAmelCase_												=       []
      # If the number of encoder and decoder layers are present in the model configuration, both are considered
      UpperCAmelCase_							,							UpperCAmelCase_												=       self.num_layers
      UpperCAmelCase_												=       min(__lowerCAmelCase							,  __lowerCAmelCase					)
      UpperCAmelCase_												=       max(__lowerCAmelCase							,  __lowerCAmelCase					) - min_num_layers
      UpperCAmelCase_												=       "encoder" if num_encoder_layers > num_decoder_layers else "decoder"

      for _ in range(__lowerCAmelCase					):
       common_inputs["past_key_values"].append(
           (
               torch.zeros(__lowerCAmelCase					),
               torch.zeros(__lowerCAmelCase					),
               torch.zeros(__lowerCAmelCase					),
               torch.zeros(__lowerCAmelCase					),
           )					)
      # TODO: test this.
      UpperCAmelCase_												=       encoder_shape if remaining_side_name == "encoder" else decoder_shape
      for _ in range(__lowerCAmelCase							,  __lowerCAmelCase					):
       common_inputs["past_key_values"].append((torch.zeros(__lowerCAmelCase					), torch.zeros(__lowerCAmelCase					))					)
     return common_inputs







    def 	lowerCamelCase_     (						self							,  UpperCamelCase__							,  UpperCamelCase__ = -1							,  UpperCamelCase__ = -1							,  UpperCamelCase__ = False							,  UpperCamelCase__ = None							,  )					->							List[Any]:





     """simple docstring"""

     UpperCAmelCase_												=       self._generate_dummy_inputs_for_encoder_and_decoder(
         __lowerCAmelCase							,  __lowerCAmelCase							,  __lowerCAmelCase							,  __lowerCAmelCase							,  __lowerCAmelCase					)

     if self.use_past:
      if not is_torch_available():
       raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."					)
      else:
       import torch
      UpperCAmelCase_							,							UpperCAmelCase_												=       common_inputs["input_ids"].shape
      # Not using the same length for past_key_values
      UpperCAmelCase_												=       seqlen + 2
      UpperCAmelCase_							,							UpperCAmelCase_												=       self.num_layers
      UpperCAmelCase_							,							UpperCAmelCase_												=       self.num_attention_heads
      UpperCAmelCase_												=       (
          batch,
          num_encoder_attention_heads,
          past_key_values_length,
          self._config.hidden_size // num_encoder_attention_heads,
      )

      UpperCAmelCase_												=       common_inputs["attention_mask"].dtype
      UpperCAmelCase_												=       torch.cat(
          [common_inputs["attention_mask"], torch.ones(__lowerCAmelCase							,  __lowerCAmelCase							,  dtype=__lowerCAmelCase					)]							,  dim=1					)
      UpperCAmelCase_												=       [
          (torch.zeros(__lowerCAmelCase					), torch.zeros(__lowerCAmelCase					)) for _ in range(__lowerCAmelCase					)
      ]
     return common_inputs







    def 	lowerCamelCase_     (						self							,  UpperCamelCase__							,  UpperCamelCase__ = -1							,  UpperCamelCase__ = -1							,  UpperCamelCase__ = False							,  UpperCamelCase__ = None							,  )					->							Any:





     """simple docstring"""

     UpperCAmelCase_												=       compute_effective_axis_dimension(
         __lowerCAmelCase							,  fixed_dimension=OnnxConfig.default_fixed_batch							,  num_token_to_add=0					)

     # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
     UpperCAmelCase_												=       tokenizer.num_special_tokens_to_add(__lowerCAmelCase					)
     UpperCAmelCase_												=       compute_effective_axis_dimension(
         __lowerCAmelCase							,  fixed_dimension=OnnxConfig.default_fixed_sequence							,  num_token_to_add=__lowerCAmelCase					)

     # Generate dummy inputs according to compute batch and sequence
     UpperCAmelCase_												=       [" ".join([tokenizer.unk_token]					) * seq_length] * batch_size
     UpperCAmelCase_												=       dict(tokenizer(__lowerCAmelCase							,  return_tensors=__lowerCAmelCase					)					)
     return common_inputs







    def 	lowerCamelCase_     (						self							,  UpperCamelCase__							,  UpperCamelCase__ = -1							,  UpperCamelCase__ = -1							,  UpperCamelCase__ = False							,  UpperCamelCase__ = None							,  )					->							Optional[Any]:





     """simple docstring"""

     if self.task in ["default", "seq2seq-lm"]:
      UpperCAmelCase_												=       self._generate_dummy_inputs_for_default_and_seqaseq_lm(
          __lowerCAmelCase							,  batch_size=__lowerCAmelCase							,  seq_length=__lowerCAmelCase							,  is_pair=__lowerCAmelCase							,  framework=__lowerCAmelCase					)

     else:
      UpperCAmelCase_												=       self._generate_dummy_inputs_for_causal_lm(
          __lowerCAmelCase							,  batch_size=__lowerCAmelCase							,  seq_length=__lowerCAmelCase							,  is_pair=__lowerCAmelCase							,  framework=__lowerCAmelCase					)

     return common_inputs







    def 	lowerCamelCase_     (						self							,  UpperCamelCase__							,  UpperCamelCase__							,  UpperCamelCase__							,  UpperCamelCase__					)					->							Any:





     """simple docstring"""

     if self.task in ["default", "seq2seq-lm"]:
      UpperCAmelCase_												=       super()._flatten_past_key_values_(__lowerCAmelCase							,  __lowerCAmelCase							,  __lowerCAmelCase							,  __lowerCAmelCase					)
     else:
      UpperCAmelCase_												=       super(__lowerCAmelCase							,  self					)._flatten_past_key_values_(
          __lowerCAmelCase							,  __lowerCAmelCase							,  __lowerCAmelCase							,  __lowerCAmelCase					)







    @property
    def 	lowerCamelCase_     (						self					)					->							int:





     """simple docstring"""

     return 1e-4







# We ignore warnings about stepping the scheduler since we step it ourselves during gradient accumulation

import warnings

from .state import AcceleratorState, GradientState


warnings.filterwarnings("ignore", category=UserWarning, module="torch.optim.lr_scheduler")

class 							__SCREAMING_SNAKE_CASE       :






							def __init__(    self	,			__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase = True	,			__lowerCAmelCase = False	):
														UpperCamelCase__					  =      scheduler
														UpperCamelCase__					  =      optimizers if isinstance(__lowerCAmelCase	,			(list, tuple)	) else [optimizers]
														UpperCamelCase__					  =      split_batches
														UpperCamelCase__					  =      step_with_optimizer
														UpperCamelCase__					  =      GradientState()






							def 					_lowerCamelCase		(    self	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														if not self.step_with_optimizer:
																					# No link between scheduler and optimizer -> just step
																					self.scheduler.step(*__lowerCAmelCase	,			**__lowerCAmelCase	)
																					return

														# Otherwise, first make sure the optimizer was stepped.
														if not self.gradient_state.sync_gradients:
																					if self.gradient_state.adjust_scheduler:
																												self.scheduler._step_count += 1
																					return

														for opt in self.optimizers:
																					if opt.step_was_skipped:
																												return
														if self.split_batches:
																					# Split batches -> the training dataloader batch size is not changed so one step per training step
																					self.scheduler.step(*__lowerCAmelCase	,			**__lowerCAmelCase	)
														else:
																					# Otherwise the training dataloader batch size was multiplied by `num_processes`, so we need to do
																					# num_processes steps per training step
																					UpperCamelCase__					  =      AcceleratorState().num_processes
																					for _ in range(__lowerCAmelCase	):
																												# Special case when using OneCycle and `drop_last` was not used
																												if hasattr(self.scheduler	,			"""total_steps"""	):
																																			if self.scheduler._step_count <= self.scheduler.total_steps:
																																										self.scheduler.step(*__lowerCAmelCase	,			**__lowerCAmelCase	)
																												else:
																																			self.scheduler.step(*__lowerCAmelCase	,			**__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	):
														return self.scheduler.get_last_lr()






							def 					_lowerCamelCase		(    self	):
														return self.scheduler.state_dict()






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														self.scheduler.load_state_dict(__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	):
														return self.scheduler.get_lr()







							def 					_lowerCamelCase		(    self	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														return self.scheduler.print_lr(*__lowerCAmelCase	,			**__lowerCAmelCase	)




