
import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map






def   lowerCamelCase__							(					a      )      ->   List[Tuple[int, ...]]:
   _A:  Dict					=       []
   if isinstance(a		,		a      ):
      for v in tree.values():
         shapes.extend(_fetch_dims(a      )      )
   elif isinstance(a		,		(list, tuple)      ):
      for t in tree:
         shapes.extend(_fetch_dims(a      )      )
   elif isinstance(a		,		torch.Tensor      ):
      shapes.append(tree.shape      )
   else:
      raise ValueError('''Not supported'''      )

   return shapes






@torch.jit.ignore
def   lowerCamelCase__							(					a		,		a      )      ->   Tuple[int, ...]:
   _A:  Any					=       []
   for d in reversed(a      ):
      idx.append(flat_idx % d      )
      _A:  Optional[int]					=       flat_idx // d

   return tuple(reversed(a      )      )






@torch.jit.ignore
def   lowerCamelCase__							(					a		,		a		,		a		,		a = None		,		a = None		,		)      ->   List[Tuple[slice, ...]]:

   # start_edges and end_edges both indicate whether, starting from any given
   # dimension, the start/end index is at the top/bottom edge of the
   # corresponding tensor, modeled as a tree
   def reduce_edge_list(a      ) -> None:
      _A:  Any					=       True
      for i in range(len(a      )      ):
         _A:  List[Any]					=       -1 * (i + 1)
         l[reversed_idx] &= tally
         _A:  int					=       l[reversed_idx]

   if start_edges is None:
      _A:  str					=       [s == 0 for s in start]
      reduce_edge_list(a      )
   if end_edges is None:
      _A:  str					=       [e == (d - 1) for e, d in zip(a		,		a      )]
      reduce_edge_list(a      )

   # Base cases. Either start/end are empty and we're done, or the final,
   # one-dimensional tensor can be simply sliced
   if len(a      ) == 0:
      return [()]
   elif len(a      ) == 1:
      return [(slice(start[0]		,		end[0] + 1      ),)]

   _A:  List[Tuple[slice, ...]]					=       []
   _A:  List[slice]					=       []

   # Dimensions common to start and end can be selected directly
   for s, e in zip(a		,		a      ):
      if s == e:
         path_list.append(slice(a		,		s + 1      )      )
      else:
         break

   _A:  Tuple[slice, ...]					=       tuple(a      )
   _A:  Optional[int]					=       len(a      )

   # start == end, and we're done
   if divergence_idx == len(a      ):
      return [path]

   def upper() -> Tuple[Tuple[slice, ...], ...]:
      assert start_edges is not None
      assert end_edges is not None

      _A:  Tuple					=       start[divergence_idx]
      return tuple(
          path + (slice(a		,		sdi + 1      ),) + s
          for s in _get_minimal_slice_set(
              start[divergence_idx + 1 :]		,		[d - 1 for d in dims[divergence_idx + 1 :]]		,		dims[divergence_idx + 1 :]		,		start_edges=start_edges[divergence_idx + 1 :]		,		end_edges=[True for _ in end_edges[divergence_idx + 1 :]]		,		)      )

   def lower() -> Tuple[Tuple[slice, ...], ...]:
      assert start_edges is not None
      assert end_edges is not None

      _A:  Optional[int]					=       end[divergence_idx]
      return tuple(
          path + (slice(a		,		edi + 1      ),) + s
          for s in _get_minimal_slice_set(
              [0 for _ in start[divergence_idx + 1 :]]		,		end[divergence_idx + 1 :]		,		dims[divergence_idx + 1 :]		,		start_edges=[True for _ in start_edges[divergence_idx + 1 :]]		,		end_edges=end_edges[divergence_idx + 1 :]		,		)      )

   # If both start and end are at the edges of the subtree rooted at
   # divergence_idx, we can just select the whole subtree at once
   if start_edges[divergence_idx] and end_edges[divergence_idx]:
      slices.append(path + (slice(start[divergence_idx]		,		end[divergence_idx] + 1      ),)      )
   # If just start is at the edge, we can grab almost all of the subtree,
   # treating only the ragged bottom edge as an edge case
   elif start_edges[divergence_idx]:
      slices.append(path + (slice(start[divergence_idx]		,		end[divergence_idx]      ),)      )
      slices.extend(lower()      )
   # Analogous to the previous case, but the top is ragged this time
   elif end_edges[divergence_idx]:
      slices.extend(upper()      )
      slices.append(path + (slice(start[divergence_idx] + 1		,		end[divergence_idx] + 1      ),)      )
   # If both sides of the range are ragged, we need to handle both sides
   # separately. If there's contiguous meat in between them, we can index it
   # in one big chunk
   else:
      slices.extend(upper()      )
      _A:  List[str]					=       end[divergence_idx] - start[divergence_idx]
      if middle_ground > 1:
         slices.append(path + (slice(start[divergence_idx] + 1		,		end[divergence_idx]      ),)      )
      slices.extend(lower()      )

   return slices






@torch.jit.ignore
def   lowerCamelCase__							(					a		,		a		,		a		,		a      )      ->   torch.Tensor:
   _A:  Union[str, Any]					=       t.shape[:no_batch_dims]
   _A:  List[Any]					=       list(_flat_idx_to_idx(a		,		a      )      )
   # _get_minimal_slice_set is inclusive
   _A:  Tuple					=       list(_flat_idx_to_idx(flat_end - 1		,		a      )      )

   # Get an ordered list of slices to perform
   _A:  Tuple					=       _get_minimal_slice_set(
       a		,		a		,		a		,		)

   _A:  str					=       [t[s] for s in slices]

   return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]      ) for s in sliced_tensors]      )






def   lowerCamelCase__							(					a		,		a		,		a		,		a		,		a = False		,		a = None		,		a = False		,		)      ->   Any:
   if not (len(a      ) > 0):
      raise ValueError('''Must provide at least one input'''      )

   _A:  int					=       [shape[:no_batch_dims] for shape in _fetch_dims(a      )]
   _A:  Tuple					=       tuple([max(a      ) for s in zip(*a      )]      )

   def _prep_inputs(a      ) -> torch.Tensor:
      if not low_mem:
         if not sum(t.shape[:no_batch_dims]      ) == no_batch_dims:
            _A:  int					=       t.expand(orig_batch_dims + t.shape[no_batch_dims:]      )
         _A:  Optional[Any]					=       t.reshape(-1		,		*t.shape[no_batch_dims:]      )
      else:
         _A:  str					=       t.expand(orig_batch_dims + t.shape[no_batch_dims:]      )
      return t

   _A:  Dict[str, Any]					=       tensor_tree_map(_prep_inputs		,		a      )
   _A:  Optional[Any]					=       None
   if _out is not None:
      _A:  int					=       tensor_tree_map(lambda a      : t.view([-1] + list(t.shape[no_batch_dims:]      )      )		,		_out      )

   _A:  Optional[Any]					=       1
   for d in orig_batch_dims:
      flat_batch_dim *= d

   _A:  List[Any]					=       flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

   def _select_chunk(a      ) -> torch.Tensor:
      return t[i : i + chunk_size] if t.shape[0] != 1 else t

   _A:  Optional[Any]					=       0
   _A:  str					=       prepped_outputs
   for _ in range(a      ):
      # Chunk the input
      if not low_mem:
         _A:  Union[str, Any]					=       _select_chunk
      else:
         _A:  Tuple					=       partial(
             _chunk_slice		,		flat_start=a		,		flat_end=min(a		,		i + chunk_size      )		,		no_batch_dims=len(a      )		,		)

      _A:  Dict[str, Any]					=       tensor_tree_map(a		,		a      )

      # Run the layer on the chunk
      _A:  List[str]					=       layer(**a      )

      # Allocate space for the output
      if out is None:
         _A:  List[Any]					=       tensor_tree_map(lambda a      : t.new_zeros((flat_batch_dim,) + t.shape[1:]      )		,		a      )

      # Put the chunk in its pre-allocated space
      if isinstance(a		,		a      ):

         def assign(a		,		a      ) -> None:
            for k, v in da.items():
               if isinstance(a		,		a      ):
                  assign(a		,		da[k]      )
               else:
                  if _add_into_out:
                     v[i : i + chunk_size] += da[k]
                  else:
                     _A:  Optional[Any]					=       da[k]

         assign(a		,		a      )
      elif isinstance(a		,		a      ):
         for xa, xa in zip(a		,		a      ):
            if _add_into_out:
               xa[i : i + chunk_size] += xa
            else:
               _A:  Dict					=       xa
      elif isinstance(a		,		torch.Tensor      ):
         if _add_into_out:
            out[i : i + chunk_size] += output_chunk
         else:
            _A:  int					=       output_chunk
      else:
         raise ValueError('''Not supported'''      )

      i += chunk_size

   _A:  Any					=       tensor_tree_map(lambda a      : t.view(orig_batch_dims + t.shape[1:]      )		,		a      )

   return out






class UpperCAmelCase   :





   '''simple docstring'''


   def __init__(					self     :							int			,			lowerCAmelCase_     :							int = 5_1_2			,			):

      """simple docstring"""
      _A:  List[str]					=       max_chunk_size
      _A:  Optional[int]					=       None
      _A:  Optional[tuple]					=       None


   def  __magic_name__				(					self     :							Optional[Any]			,			lowerCAmelCase_     :							Callable			,			lowerCAmelCase_     :							tuple			,			lowerCAmelCase_     :							int     ):

      """simple docstring"""
      logging.info('''Tuning chunk size...'''     )

      if min_chunk_size >= self.max_chunk_size:
         return min_chunk_size

      _A:  List[int]					=       [2**l for l in range(int(math.log(self.max_chunk_size			,			2     )     ) + 1     )]
      _A:  List[str]					=       [c for c in candidates if c > min_chunk_size]
      _A:  Union[str, Any]					=       [min_chunk_size] + candidates
      candidates[-1] += 4

      def test_chunk_size(lowerCAmelCase_     :							int     ) -> bool:
         try:
            with torch.no_grad():
               fn(*lowerCAmelCase_			,			chunk_size=lowerCAmelCase_     )
            return True
         except RuntimeError:
            return False

      _A:  Dict					=       0
      _A:  Any					=       len(lowerCAmelCase_     ) - 1
      while i > min_viable_chunk_size_index:
         _A:  Optional[Any]					=       test_chunk_size(candidates[i]     )
         if not viable:
            _A:  Optional[Any]					=       (min_viable_chunk_size_index + i) // 2
         else:
            _A:  Optional[int]					=       i
            _A:  Dict					=       (i + len(lowerCAmelCase_     ) - 1) // 2

      return candidates[min_viable_chunk_size_index]


   def  __magic_name__				(					self     :							Optional[int]			,			lowerCAmelCase_     :							Iterable			,			lowerCAmelCase_     :							Iterable     ):

      """simple docstring"""
      _A:  Optional[int]					=       True
      for aa, aa in zip(lowerCAmelCase_			,			lowerCAmelCase_     ):
         assert type(lowerCAmelCase_     ) == type(lowerCAmelCase_     )
         if isinstance(lowerCAmelCase_			,			(list, tuple)     ):
            consistent &= self._compare_arg_caches(lowerCAmelCase_			,			lowerCAmelCase_     )
         elif isinstance(lowerCAmelCase_			,			lowerCAmelCase_     ):
            _A:  Optional[Any]					=       [v for _, v in sorted(aa.items()			,			key=lambda lowerCAmelCase_     : x[0]     )]
            _A:  Any					=       [v for _, v in sorted(aa.items()			,			key=lambda lowerCAmelCase_     : x[0]     )]
            consistent &= self._compare_arg_caches(lowerCAmelCase_			,			lowerCAmelCase_     )
         else:
            consistent &= aa == aa

      return consistent


   def  __magic_name__				(					self     :							str			,			lowerCAmelCase_     :							Callable			,			lowerCAmelCase_     :							tuple			,			lowerCAmelCase_     :							int			,			):

      """simple docstring"""
      _A:  Union[str, Any]					=       True
      _A:  tuple					=       tree_map(lambda lowerCAmelCase_     : a.shape if isinstance(lowerCAmelCase_			,			torch.Tensor     ) else a			,			lowerCAmelCase_			,			lowerCAmelCase_     )
      if self.cached_arg_data is not None:
         # If args have changed shape/value, we need to re-tune
         assert len(self.cached_arg_data     ) == len(lowerCAmelCase_     )
         _A:  str					=       self._compare_arg_caches(self.cached_arg_data			,			lowerCAmelCase_     )
      else:
         # Otherwise, we can reuse the precomputed value
         _A:  Optional[int]					=       False

      if not consistent:
         _A:  Tuple					=       self._determine_favorable_chunk_size(
             lowerCAmelCase_			,			lowerCAmelCase_			,			lowerCAmelCase_			,			)
         _A:  str					=       arg_data

      assert self.cached_chunk_size is not None

      return self.cached_chunk_size





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCAmelCase__		:     List[str]     			=      logging.get_logger(__name__)

UpperCAmelCase__		:     Union[str, Any]     			=      {
    'xlm-mlm-en-2048': 'https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json',
    'xlm-mlm-ende-1024': 'https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json',
    'xlm-mlm-enfr-1024': 'https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json',
    'xlm-mlm-enro-1024': 'https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json',
    'xlm-mlm-tlm-xnli15-1024': 'https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json',
    'xlm-mlm-xnli15-1024': 'https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json',
    'xlm-clm-enfr-1024': 'https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json',
    'xlm-clm-ende-1024': 'https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json',
    'xlm-mlm-17-1280': 'https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json',
    'xlm-mlm-100-1280': 'https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json',
}






class UpperCAmelCase   (						SCREAMING_SNAKE_CASE__   ):





   '''simple docstring'''
   __UpperCamelCase		:    int 				=    '''xlm'''
   __UpperCamelCase		:    Optional[Any] 				=    {
       '''hidden_size''': '''emb_dim''',
       '''num_attention_heads''': '''n_heads''',
       '''num_hidden_layers''': '''n_layers''',
       '''n_words''': '''vocab_size''',  # For backward compatibility
   }


   def __init__(					self     :							List[Any]			,			lowerCAmelCase_     :							Dict=3_0_1_4_5			,			lowerCAmelCase_     :							Union[str, Any]=2_0_4_8			,			lowerCAmelCase_     :							List[str]=1_2			,			lowerCAmelCase_     :							Optional[Any]=1_6			,			lowerCAmelCase_     :							Union[str, Any]=0.1			,			lowerCAmelCase_     :							List[str]=0.1			,			lowerCAmelCase_     :							Optional[int]=True			,			lowerCAmelCase_     :							List[str]=False			,			lowerCAmelCase_     :							Tuple=False			,			lowerCAmelCase_     :							List[str]=False			,			lowerCAmelCase_     :							Optional[Any]=1			,			lowerCAmelCase_     :							List[str]=True			,			lowerCAmelCase_     :							Tuple=5_1_2			,			lowerCAmelCase_     :							Tuple=2_0_4_8**-0.5			,			lowerCAmelCase_     :							List[str]=1e-12			,			lowerCAmelCase_     :							Any=0.02			,			lowerCAmelCase_     :							int=0			,			lowerCAmelCase_     :							int=1			,			lowerCAmelCase_     :							Dict=2			,			lowerCAmelCase_     :							List[Any]=3			,			lowerCAmelCase_     :							int=5			,			lowerCAmelCase_     :							Optional[int]=True			,			lowerCAmelCase_     :							Union[str, Any]="first"			,			lowerCAmelCase_     :							int=True			,			lowerCAmelCase_     :							List[Any]=None			,			lowerCAmelCase_     :							Any=True			,			lowerCAmelCase_     :							int=0.1			,			lowerCAmelCase_     :							Tuple=5			,			lowerCAmelCase_     :							Tuple=5			,			lowerCAmelCase_     :							Optional[int]=0			,			lowerCAmelCase_     :							Optional[Any]=0			,			lowerCAmelCase_     :							str=2			,			lowerCAmelCase_     :							Dict=0			,			**lowerCAmelCase_     :							List[Any]			,			):

      """simple docstring"""
      _A:  Dict					=       vocab_size
      _A:  Dict					=       emb_dim
      _A:  Optional[Any]					=       n_layers
      _A:  str					=       n_heads
      _A:  Optional[int]					=       dropout
      _A:  Union[str, Any]					=       attention_dropout
      _A:  Any					=       gelu_activation
      _A:  Optional[Any]					=       sinusoidal_embeddings
      _A:  Any					=       causal
      _A:  Optional[int]					=       asm
      _A:  List[str]					=       n_langs
      _A:  int					=       use_lang_emb
      _A:  Any					=       layer_norm_eps
      _A:  Tuple					=       bos_index
      _A:  Any					=       eos_index
      _A:  List[str]					=       pad_index
      _A:  str					=       unk_index
      _A:  str					=       mask_index
      _A:  List[Any]					=       is_encoder
      _A:  List[Any]					=       max_position_embeddings
      _A:  str					=       embed_init_std
      _A:  int					=       init_std
      _A:  List[str]					=       summary_type
      _A:  List[str]					=       summary_use_proj
      _A:  Tuple					=       summary_activation
      _A:  Dict					=       summary_proj_to_labels
      _A:  Tuple					=       summary_first_dropout
      _A:  str					=       start_n_top
      _A:  str					=       end_n_top
      _A:  Optional[Any]					=       mask_token_id
      _A:  Any					=       lang_id

      if "n_words" in kwargs:
         _A:  Union[str, Any]					=       kwargs['''n_words''']

      super().__init__(pad_token_id=lowerCAmelCase_			,			bos_token_id=lowerCAmelCase_			,			**lowerCAmelCase_     )






class UpperCAmelCase   (						SCREAMING_SNAKE_CASE__   ):





   '''simple docstring'''


   @property
   def  __magic_name__				(					self     :							List[str]     ):

      """simple docstring"""
      if self.task == "multiple-choice":
         _A:  int					=       {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
      else:
         _A:  Tuple					=       {0: '''batch''', 1: '''sequence'''}
      return OrderedDict(
          [
              ('''input_ids''', dynamic_axis),
              ('''attention_mask''', dynamic_axis),
              ('''token_type_ids''', dynamic_axis),
          ]     )



