codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license convert robertaprelayernorm checkpoint import argparse import torch from huggingfacehub import hfhubdownload from transformers import autotokenizer robertaprelayernormconfig robertaprelayernormformaskedlm from transformers utils import logging logging setverbosityinfo logger logging getloggername def convertrobertaprelayernormcheckpointtopytorchcheckpointrepo str pytorchdumpfolderpath str convert configuration config robertaprelayernormconfig frompretrained checkpointrepo architecturesrobertaprelayernormformaskedlm convert statedict originalstatedict torch loadhfhubdownloadrepoidcheckpointrepo filenamepytorchmodel bin statedict for tensorkey tensorvalue in originalstatedict items the transformer implementation gives the model a unique name rather than overwiriting roberta if tensorkey startswithroberta tensorkey robertaprelayernorm tensorkeylenroberta the original implementation contains weights which are not used remove them from the statedict if tensorkey endswith self layernorm weight or tensorkey endswith self layernorm bias continue statedicttensorkey tensorvalue model robertaprelayernormformaskedlm frompretrained pretrainedmodelnameorpathnone configconfig statedictstatedict model savepretrainedpytorchdumpfolderpath convert tokenizer tokenizer autotokenizer frompretrainedcheckpointrepo tokenizer savepretrainedpytorchdumpfolderpath if name main parser argparse argumentparser required parameters parser addargument checkpointrepo defaultnone typestr requiredtrue helppath the official pytorch dump e g andreasmadsenefficientmlmm0 40 parser addargument pytorchdumpfolderpath defaultnone typestr requiredtrue helppath to the output pytorch model args parser parseargs convertrobertaprelayernormcheckpointtopytorchargs checkpointrepo args pytorchdumpfolderpath coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license convert roberta prelayernorm checkpoint copy paste tweak roberta_prelayernorm s weights to our bert structure convert configuration convert state_dict the transformer implementation gives the model a unique name rather than overwiriting roberta the original implementation contains weights which are not used remove them from the state_dict convert tokenizer required parameters
import argparse

import torch
from huggingface_hub import hf_hub_download

from transformers import AutoTokenizer, RobertaPreLayerNormConfig, RobertaPreLayerNormForMaskedLM
from transformers.utils import logging


logging.set_verbosity_info()
logger = logging.get_logger(__name__)


def convert_roberta_prelayernorm_checkpoint_to_pytorch(checkpoint_repo: str, pytorch_dump_folder_path: str):
    
    
    config = RobertaPreLayerNormConfig.from_pretrained(
        checkpoint_repo, architectures=["RobertaPreLayerNormForMaskedLM"]
    )

    
    original_state_dict = torch.load(hf_hub_download(repo_id=checkpoint_repo, filename="pytorch_model.bin"))
    state_dict = {}
    for tensor_key, tensor_value in original_state_dict.items():
        
        if tensor_key.startswith("roberta."):
            tensor_key = "roberta_prelayernorm." + tensor_key[len("roberta.") :]

        
        if tensor_key.endswith(".self.LayerNorm.weight") or tensor_key.endswith(".self.LayerNorm.bias"):
            continue

        state_dict[tensor_key] = tensor_value

    model = RobertaPreLayerNormForMaskedLM.from_pretrained(
        pretrained_model_name_or_path=None, config=config, state_dict=state_dict
    )
    model.save_pretrained(pytorch_dump_folder_path)

    
    tokenizer = AutoTokenizer.from_pretrained(checkpoint_repo)
    tokenizer.save_pretrained(pytorch_dump_folder_path)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    
    parser.add_argument(
        "--checkpoint-repo",
        default=None,
        type=str,
        required=True,
        help="Path the official PyTorch dump, e.g. 'andreasmadsen/efficient_mlm_m0.40'.",
    )
    parser.add_argument(
        "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
    )
    args = parser.parse_args()
    convert_roberta_prelayernorm_checkpoint_to_pytorch(args.checkpoint_repo, args.pytorch_dump_folder_path)