






from collections.abc import Generator
from math import sin



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      bytes     )    ->   bytes:





    """simple docstring"""

    if len(__lowercase     ) != 3_2:
        raise ValueError("""Input must be of length 32"""     )

    __A       =						B""""""
    for i in [3, 2, 1, 0]:
        little_endian += string_aa[8 * i : 8 * i + 8]
    return little_endian



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      int     )    ->   bytes:





    """simple docstring"""

    if i < 0:
        raise ValueError("""Input must be non-negative"""     )

    __A       =						format(__lowercase      ,					"""08x"""     )[-8:]
    __A       =						B""""""
    for i in [3, 2, 1, 0]:
        little_endian_hex += hex_rep[2 * i : 2 * i + 2].encode("""utf-8"""     )
    return little_endian_hex



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      bytes     )    ->   bytes:





    """simple docstring"""

    __A       =						B""""""
    for char in message:
        bit_string += format(__lowercase      ,					"""08b"""     ).encode("""utf-8"""     )
    __A       =						format(len(__lowercase     )      ,					"""064b"""     ).encode("""utf-8"""     )

    # Pad bit_string to a multiple of 512 chars
    bit_string += b"1"
    while len(__lowercase     ) % 5_1_2 != 4_4_8:
        bit_string += b"0"
    bit_string += to_little_endian(start_len[3_2:]     ) + to_little_endian(start_len[:3_2]     )

    return bit_string



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      bytes     )    ->   Generator[list[int], None, None]:





    """simple docstring"""

    if len(__lowercase     ) % 5_1_2 != 0:
        raise ValueError("""Input must have length that's a multiple of 512"""     )

    for pos in range(0      ,					len(__lowercase     )      ,					5_1_2     ):
        __A       =						bit_string[pos : pos + 5_1_2]
        __A       =						[]
        for i in range(0      ,					5_1_2      ,					3_2     ):
            block_words.append(int(to_little_endian(block[i : i + 3_2]     )      ,					2     )     )
        yield block_words



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      int     )    ->   int:





    """simple docstring"""

    if i < 0:
        raise ValueError("""Input must be non-negative"""     )

    __A       =						format(__lowercase      ,					"""032b"""     )
    __A       =						""""""
    for c in i_str:
        new_str += "1" if c == "0" else "0"
    return int(__lowercase      ,					2     )



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      int      ,					__lowercase :      int     )    ->   int:





    """simple docstring"""

    return (a + b) % 2**3_2



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      int      ,					__lowercase :      int     )    ->   int:





    """simple docstring"""

    if i < 0:
        raise ValueError("""Input must be non-negative"""     )
    if shift < 0:
        raise ValueError("""Shift must be non-negative"""     )
    return ((i << shift) ^ (i >> (3_2 - shift))) % 2**3_2



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      bytes     )    ->   bytes:





    """simple docstring"""

    __A       =						preprocess(__lowercase     )

    __A       =						[int(2**3_2 * abs(sin(i + 1     )     )     ) for i in range(6_4     )]

    # Starting states
    __A       =						0X6_7_4_5_2_3_0_1
    __A       =						0Xe_f_c_d_a_b_8_9
    __A       =						0X9_8_b_a_d_c_f_e
    __A       =						0X1_0_3_2_5_4_7_6

    __A       =						[
        7,
        1_2,
        1_7,
        2_2,
        7,
        1_2,
        1_7,
        2_2,
        7,
        1_2,
        1_7,
        2_2,
        7,
        1_2,
        1_7,
        2_2,
        5,
        9,
        1_4,
        2_0,
        5,
        9,
        1_4,
        2_0,
        5,
        9,
        1_4,
        2_0,
        5,
        9,
        1_4,
        2_0,
        4,
        1_1,
        1_6,
        2_3,
        4,
        1_1,
        1_6,
        2_3,
        4,
        1_1,
        1_6,
        2_3,
        4,
        1_1,
        1_6,
        2_3,
        6,
        1_0,
        1_5,
        2_1,
        6,
        1_0,
        1_5,
        2_1,
        6,
        1_0,
        1_5,
        2_1,
        6,
        1_0,
        1_5,
        2_1,
    ]

    # Process bit string in chunks, each with 16 32-char words
    for block_words in get_block_words(__lowercase     ):
        __A       =						aa
        __A       =						ba
        __A       =						ca
        __A       =						da

        # Hash current chunk
        for i in range(6_4     ):
            if i <= 1_5:
                # f = (b & c) | (not_32(b) & d)     # Alternate definition for f
                __A       =						d ^ (b & (c ^ d))
                __A       =						i
            elif i <= 3_1:
                # f = (d & b) | (not_32(d) & c)     # Alternate definition for f
                __A       =						c ^ (d & (b ^ c))
                __A       =						(5 * i + 1) % 1_6
            elif i <= 4_7:
                __A       =						b ^ c ^ d
                __A       =						(3 * i + 5) % 1_6
            else:
                __A       =						c ^ (b | not_aa(__lowercase     ))
                __A       =						(7 * i) % 1_6
            __A       =						(f + a + added_consts[i] + block_words[g]) % 2**3_2
            __A       =						d
            __A       =						c
            __A       =						b
            __A       =						sum_aa(__lowercase      ,					left_rotate_aa(__lowercase      ,					shift_amounts[i]     )     )

        # Add hashed chunk to running total
        __A       =						sum_aa(__lowercase      ,					__lowercase     )
        __A       =						sum_aa(__lowercase      ,					__lowercase     )
        __A       =						sum_aa(__lowercase      ,					__lowercase     )
        __A       =						sum_aa(__lowercase      ,					__lowercase     )

    __A       =						reformat_hex(__lowercase     ) + reformat_hex(__lowercase     ) + reformat_hex(__lowercase     ) + reformat_hex(__lowercase     )
    return digest


if __name__ == "__main__":
  import doctest

  doctest.testmod()












import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse("1.0.0a"):
  raise Exception("requires fairseq >= 1.0.0a")

logging.set_verbosity_info()
__a   :							Union[str, Any]						= logging.get_logger(__name__)

__a   :							Tuple						= "Hello world! cécé herlolip"



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      str      ,					__lowercase :      str      ,					__lowercase :      bool     )    ->   List[Any]:





    """simple docstring"""

    __A       =						FairseqRobertaModel.from_pretrained(__lowercase     )
    roberta.eval()  # disable dropout
    __A       =						roberta.model.encoder.sentence_encoder
    __A       =						XLMRobertaConfig(
        vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings      ,					hidden_size=roberta.cfg.model.encoder_embed_dim      ,					num_hidden_layers=roberta.cfg.model.encoder_layers      ,					num_attention_heads=roberta.cfg.model.encoder_attention_heads      ,					intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim      ,					max_position_embeddings=5_1_4      ,					type_vocab_size=1      ,					layer_norm_eps=1E-5      ,					)
    if classification_head:
        __A       =						roberta.model.classification_heads["""mnli"""].out_proj.weight.shape[0]

    print("""Our RoBERTa config:"""      ,					__lowercase     )

    __A       =						XLMRobertaXLForSequenceClassification(__lowercase     ) if classification_head else XLMRobertaXLForMaskedLM(__lowercase     )
    model.eval()

    # Now let's copy all the weights.
    # Embeddings
    __A       =						roberta_sent_encoder.embed_tokens.weight
    __A       =						roberta_sent_encoder.embed_positions.weight
    __A       =						torch.zeros_like(
        model.roberta.embeddings.token_type_embeddings.weight     )  # just zero them out b/c RoBERTa doesn't use them.

    __A       =						roberta_sent_encoder.layer_norm.weight
    __A       =						roberta_sent_encoder.layer_norm.bias

    for i in range(config.num_hidden_layers     ):
        # Encoder: start of layer
        __A       =						model.roberta.encoder.layer[i]
        __A       =						roberta_sent_encoder.layers[i]

        __A       =						layer.attention
        __A       =						roberta_layer.self_attn_layer_norm.weight
        __A       =						roberta_layer.self_attn_layer_norm.bias

        # self attention
        __A       =						layer.attention.self
        assert (
            roberta_layer.self_attn.k_proj.weight.data.shape
            == roberta_layer.self_attn.q_proj.weight.data.shape
            == roberta_layer.self_attn.v_proj.weight.data.shape
            == torch.Size((config.hidden_size, config.hidden_size)     )
        )

        __A       =						roberta_layer.self_attn.q_proj.weight
        __A       =						roberta_layer.self_attn.q_proj.bias
        __A       =						roberta_layer.self_attn.k_proj.weight
        __A       =						roberta_layer.self_attn.k_proj.bias
        __A       =						roberta_layer.self_attn.v_proj.weight
        __A       =						roberta_layer.self_attn.v_proj.bias

        # self-attention output
        __A       =						layer.attention.output
        assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
        __A       =						roberta_layer.self_attn.out_proj.weight
        __A       =						roberta_layer.self_attn.out_proj.bias

        # this one is final layer norm
        __A       =						roberta_layer.final_layer_norm.weight
        __A       =						roberta_layer.final_layer_norm.bias

        # intermediate
        __A       =						layer.intermediate
        assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
        __A       =						roberta_layer.fca.weight
        __A       =						roberta_layer.fca.bias

        # output
        __A       =						layer.output
        assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
        __A       =						roberta_layer.fca.weight
        __A       =						roberta_layer.fca.bias
        # end of layer

    if classification_head:
        __A       =						roberta.model.classification_heads["""mnli"""].dense.weight
        __A       =						roberta.model.classification_heads["""mnli"""].dense.bias
        __A       =						roberta.model.classification_heads["""mnli"""].out_proj.weight
        __A       =						roberta.model.classification_heads["""mnli"""].out_proj.bias
    else:
        # LM Head
        __A       =						roberta.model.encoder.lm_head.dense.weight
        __A       =						roberta.model.encoder.lm_head.dense.bias
        __A       =						roberta.model.encoder.lm_head.layer_norm.weight
        __A       =						roberta.model.encoder.lm_head.layer_norm.bias
        __A       =						roberta.model.encoder.lm_head.weight
        __A       =						roberta.model.encoder.lm_head.bias

    # Let's check that we get the same results.
    __A       =						roberta.encode(__lowercase     ).unsqueeze(0     )  # batch of size 1

    __A       =						model(__lowercase     )[0]
    if classification_head:
        __A       =						roberta.model.classification_heads["""mnli"""](roberta.extract_features(__lowercase     )     )
    else:
        __A       =						roberta.model(__lowercase     )[0]
    print(our_output.shape      ,					their_output.shape     )
    __A       =						torch.max(torch.abs(our_output - their_output     )     ).item()
    print(f"max_absolute_diff = {max_absolute_diff}"     )  # ~ 1e-7
    __A       =						torch.allclose(__lowercase      ,					__lowercase      ,					atol=1E-3     )
    print("""Do both models output the same tensors?"""      ,					"""🔥""" if success else """💩"""     )
    if not success:
        raise Exception("""Something went wRoNg"""     )

    pathlib.Path(__lowercase     ).mkdir(parents=__lowercase      ,					exist_ok=__lowercase     )
    print(f"Saving model to {pytorch_dump_folder_path}"     )
    model.save_pretrained(__lowercase     )


if __name__ == "__main__":
  __a   :							List[Any]						= argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      "--roberta_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump."
  )
  parser.add_argument(
      "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
  )
  parser.add_argument(
      "--classification_head", action="store_true", help="Whether to convert a final classification head."
  )
  __a   :							Dict						= parser.parse_args()
  convert_xlm_roberta_xl_checkpoint_to_pytorch(
      args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
  )




