






import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__a   :							Optional[int]						= logging.get_logger(__name__)

__a   :							Optional[Any]						= {
    "microsoft/unispeech-sat-base-100h-libri-ft": (
        "https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft/resolve/main/config.json"
    ),
    # See all UniSpeechSat models at https://huggingface.co/models?filter=unispeech_sat
}


class    __lowercase   (						lowercase_	):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE     =					"unispeech-sat"




     def __init__(					self							:      List[str]			,						UpperCamelCase_							:      Any=32			,						UpperCamelCase_							:      Tuple=768			,						UpperCamelCase_							:      Any=12			,						UpperCamelCase_							:      Union[str, Any]=12			,						UpperCamelCase_							:      Tuple=3_072			,						UpperCamelCase_							:      Optional[int]="gelu"			,						UpperCamelCase_							:      Tuple=0.1			,						UpperCamelCase_							:      Union[str, Any]=0.1			,						UpperCamelCase_							:      Dict=0.1			,						UpperCamelCase_							:      str=0.0			,						UpperCamelCase_							:      List[str]=0.0			,						UpperCamelCase_							:      Tuple=0.1			,						UpperCamelCase_							:      Union[str, Any]=0.1			,						UpperCamelCase_							:      Union[str, Any]=0.02			,						UpperCamelCase_							:      Tuple=1e-5			,						UpperCamelCase_							:      Union[str, Any]="group"			,						UpperCamelCase_							:      Union[str, Any]="gelu"			,						UpperCamelCase_							:      str=(512, 512, 512, 512, 512, 512, 512)			,						UpperCamelCase_							:      Tuple=(5, 2, 2, 2, 2, 2, 2)			,						UpperCamelCase_							:      List[str]=(10, 3, 3, 3, 3, 2, 2)			,						UpperCamelCase_							:      List[str]=False			,						UpperCamelCase_							:      Tuple=128			,						UpperCamelCase_							:      Optional[Any]=16			,						UpperCamelCase_							:      Optional[Any]=False			,						UpperCamelCase_							:      List[Any]=True			,						UpperCamelCase_							:      int=0.05			,						UpperCamelCase_							:      int=10			,						UpperCamelCase_							:      Optional[int]=2			,						UpperCamelCase_							:      Optional[int]=0.0			,						UpperCamelCase_							:      List[Any]=10			,						UpperCamelCase_							:      List[str]=0			,						UpperCamelCase_							:      Union[str, Any]=320			,						UpperCamelCase_							:      Dict=2			,						UpperCamelCase_							:      Tuple=0.1			,						UpperCamelCase_							:      Union[str, Any]=100			,						UpperCamelCase_							:      str=256			,						UpperCamelCase_							:      str=256			,						UpperCamelCase_							:      Tuple=0.1			,						UpperCamelCase_							:      List[Any]="mean"			,						UpperCamelCase_							:      Union[str, Any]=False			,						UpperCamelCase_							:      Any=False			,						UpperCamelCase_							:      List[str]=256			,						UpperCamelCase_							:      Tuple=(512, 512, 512, 512, 1_500)			,						UpperCamelCase_							:      Any=(5, 3, 3, 1, 1)			,						UpperCamelCase_							:      Dict=(1, 2, 3, 1, 1)			,						UpperCamelCase_							:      Optional[int]=512			,						UpperCamelCase_							:      Union[str, Any]=0			,						UpperCamelCase_							:      List[str]=1			,						UpperCamelCase_							:      Optional[Any]=2			,						UpperCamelCase_							:      Dict=504			,						**UpperCamelCase_							:      Any			,						):


         """simple docstring"""



         super().__init__(**UpperCamelCase_			,						pad_token_id=UpperCamelCase_			,						bos_token_id=UpperCamelCase_			,						eos_token_id=UpperCamelCase_					)
         __A       =						hidden_size
         __A       =						feat_extract_norm
         __A       =						feat_extract_activation
         __A       =						list(UpperCamelCase_					)
         __A       =						list(UpperCamelCase_					)
         __A       =						list(UpperCamelCase_					)
         __A       =						conv_bias
         __A       =						num_conv_pos_embeddings
         __A       =						num_conv_pos_embedding_groups
         __A       =						len(self.conv_dim					)
         __A       =						num_hidden_layers
         __A       =						intermediate_size
         __A       =						hidden_act
         __A       =						num_attention_heads
         __A       =						hidden_dropout
         __A       =						attention_dropout
         __A       =						activation_dropout
         __A       =						feat_proj_dropout
         __A       =						final_dropout
         __A       =						layerdrop
         __A       =						layer_norm_eps
         __A       =						initializer_range
         __A       =						vocab_size
         __A       =						num_clusters
         __A       =						do_stable_layer_norm
         __A       =						use_weighted_layer_sum

         if (
             (len(self.conv_stride					) != self.num_feat_extract_layers)
             or (len(self.conv_kernel					) != self.num_feat_extract_layers)
             or (len(self.conv_dim					) != self.num_feat_extract_layers)
         ):
             raise ValueError(
                 """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
                 """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
                 F" {len(self.conv_dim					)}`, `len(config.conv_stride) = {len(self.conv_stride					)}`,"
                 F" `len(config.conv_kernel) = {len(self.conv_kernel					)}`."					)

         # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
         __A       =						apply_spec_augment
         __A       =						mask_time_prob
         __A       =						mask_time_length
         __A       =						mask_time_min_masks
         __A       =						mask_feature_prob
         __A       =						mask_feature_length
         __A       =						mask_feature_min_masks

         # parameters for pretraining with codevector quantized representations
         __A       =						num_codevectors_per_group
         __A       =						num_codevector_groups
         __A       =						contrastive_logits_temperature
         __A       =						feat_quantizer_dropout
         __A       =						num_negatives
         __A       =						codevector_dim
         __A       =						proj_codevector_dim
         __A       =						diversity_loss_weight

         # ctc loss
         __A       =						ctc_loss_reduction
         __A       =						ctc_zero_infinity

         # SequenceClassification-specific parameter. Feel free to ignore for other classes.
         __A       =						classifier_proj_size

         # XVector-specific parameters. Feel free to ignore for other classes.
         __A       =						list(UpperCamelCase_					)
         __A       =						list(UpperCamelCase_					)
         __A       =						list(UpperCamelCase_					)
         __A       =						xvector_output_dim





     @property
     def 		lowerCAmelCase_  (					self							:      Any					):


         """simple docstring"""



         return functools.reduce(operator.mul			,						self.conv_stride			,						1					)












import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse("1.0.0a"):
  raise Exception("requires fairseq >= 1.0.0a")

logging.set_verbosity_info()
__a   :							Union[str, Any]						= logging.get_logger(__name__)

__a   :							Tuple						= "Hello world! cécé herlolip"



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      str      ,					__lowercase :      str      ,					__lowercase :      bool     )    ->   List[Any]:





    """simple docstring"""

    __A       =						FairseqRobertaModel.from_pretrained(__lowercase     )
    roberta.eval()  # disable dropout
    __A       =						roberta.model.encoder.sentence_encoder
    __A       =						XLMRobertaConfig(
        vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings      ,					hidden_size=roberta.cfg.model.encoder_embed_dim      ,					num_hidden_layers=roberta.cfg.model.encoder_layers      ,					num_attention_heads=roberta.cfg.model.encoder_attention_heads      ,					intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim      ,					max_position_embeddings=5_1_4      ,					type_vocab_size=1      ,					layer_norm_eps=1E-5      ,					)
    if classification_head:
        __A       =						roberta.model.classification_heads["""mnli"""].out_proj.weight.shape[0]

    print("""Our RoBERTa config:"""      ,					__lowercase     )

    __A       =						XLMRobertaXLForSequenceClassification(__lowercase     ) if classification_head else XLMRobertaXLForMaskedLM(__lowercase     )
    model.eval()

    # Now let's copy all the weights.
    # Embeddings
    __A       =						roberta_sent_encoder.embed_tokens.weight
    __A       =						roberta_sent_encoder.embed_positions.weight
    __A       =						torch.zeros_like(
        model.roberta.embeddings.token_type_embeddings.weight     )  # just zero them out b/c RoBERTa doesn't use them.

    __A       =						roberta_sent_encoder.layer_norm.weight
    __A       =						roberta_sent_encoder.layer_norm.bias

    for i in range(config.num_hidden_layers     ):
        # Encoder: start of layer
        __A       =						model.roberta.encoder.layer[i]
        __A       =						roberta_sent_encoder.layers[i]

        __A       =						layer.attention
        __A       =						roberta_layer.self_attn_layer_norm.weight
        __A       =						roberta_layer.self_attn_layer_norm.bias

        # self attention
        __A       =						layer.attention.self
        assert (
            roberta_layer.self_attn.k_proj.weight.data.shape
            == roberta_layer.self_attn.q_proj.weight.data.shape
            == roberta_layer.self_attn.v_proj.weight.data.shape
            == torch.Size((config.hidden_size, config.hidden_size)     )
        )

        __A       =						roberta_layer.self_attn.q_proj.weight
        __A       =						roberta_layer.self_attn.q_proj.bias
        __A       =						roberta_layer.self_attn.k_proj.weight
        __A       =						roberta_layer.self_attn.k_proj.bias
        __A       =						roberta_layer.self_attn.v_proj.weight
        __A       =						roberta_layer.self_attn.v_proj.bias

        # self-attention output
        __A       =						layer.attention.output
        assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
        __A       =						roberta_layer.self_attn.out_proj.weight
        __A       =						roberta_layer.self_attn.out_proj.bias

        # this one is final layer norm
        __A       =						roberta_layer.final_layer_norm.weight
        __A       =						roberta_layer.final_layer_norm.bias

        # intermediate
        __A       =						layer.intermediate
        assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
        __A       =						roberta_layer.fca.weight
        __A       =						roberta_layer.fca.bias

        # output
        __A       =						layer.output
        assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
        __A       =						roberta_layer.fca.weight
        __A       =						roberta_layer.fca.bias
        # end of layer

    if classification_head:
        __A       =						roberta.model.classification_heads["""mnli"""].dense.weight
        __A       =						roberta.model.classification_heads["""mnli"""].dense.bias
        __A       =						roberta.model.classification_heads["""mnli"""].out_proj.weight
        __A       =						roberta.model.classification_heads["""mnli"""].out_proj.bias
    else:
        # LM Head
        __A       =						roberta.model.encoder.lm_head.dense.weight
        __A       =						roberta.model.encoder.lm_head.dense.bias
        __A       =						roberta.model.encoder.lm_head.layer_norm.weight
        __A       =						roberta.model.encoder.lm_head.layer_norm.bias
        __A       =						roberta.model.encoder.lm_head.weight
        __A       =						roberta.model.encoder.lm_head.bias

    # Let's check that we get the same results.
    __A       =						roberta.encode(__lowercase     ).unsqueeze(0     )  # batch of size 1

    __A       =						model(__lowercase     )[0]
    if classification_head:
        __A       =						roberta.model.classification_heads["""mnli"""](roberta.extract_features(__lowercase     )     )
    else:
        __A       =						roberta.model(__lowercase     )[0]
    print(our_output.shape      ,					their_output.shape     )
    __A       =						torch.max(torch.abs(our_output - their_output     )     ).item()
    print(f"max_absolute_diff = {max_absolute_diff}"     )  # ~ 1e-7
    __A       =						torch.allclose(__lowercase      ,					__lowercase      ,					atol=1E-3     )
    print("""Do both models output the same tensors?"""      ,					"""🔥""" if success else """💩"""     )
    if not success:
        raise Exception("""Something went wRoNg"""     )

    pathlib.Path(__lowercase     ).mkdir(parents=__lowercase      ,					exist_ok=__lowercase     )
    print(f"Saving model to {pytorch_dump_folder_path}"     )
    model.save_pretrained(__lowercase     )


if __name__ == "__main__":
  __a   :							List[Any]						= argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      "--roberta_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump."
  )
  parser.add_argument(
      "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
  )
  parser.add_argument(
      "--classification_head", action="store_true", help="Whether to convert a final classification head."
  )
  __a   :							Dict						= parser.parse_args()
  convert_xlm_roberta_xl_checkpoint_to_pytorch(
      args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
  )




