

# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig







def       lowercase					(	__A :					Union[str, Any] ,    __A :					Any ,    __A :					Any=0		)				->			Any:





    '''simple docstring'''




    if name is None:
        snake_case							:		str        =		None
    else:
        snake_case							:		int        =		""".""" * max(0 ,    spaces - 2		) + """# {:""" + str(50 - spaces		) + """s}"""
        snake_case							:		Tuple        =		fmt.format(__A		)

    # Print and recurse (if needed).
    if isinstance(__A ,    __A		):
        if msg is not None:
            print(__A		)
        for k in val.keys():
            recursive_print(__A ,    val[k] ,    spaces + 2		)
    elif isinstance(__A ,    torch.Tensor		):
        print(__A ,    """:""" ,    val.size()		)
    else:
        print(__A ,    """:""" ,    __A		)







def       lowercase					(	__A :					Any ,    __A :					Dict ,    __A :					str ,    __A :					Tuple ,    __A :					Optional[int]		)				->			List[str]:





    '''simple docstring'''




    snake_case							:		int        =		param.size()
    if checkpoint_version == 1.0:
        # version 1.0 stores [num_heads * hidden_size * num_splits, :]
        snake_case							:		List[str]        =		(num_heads, hidden_size, num_splits) + input_shape[1:]
        snake_case							:		Optional[int]        =		param.view(*__A		)
        snake_case							:		Dict        =		param.transpose(0 ,    2		)
        snake_case							:		int        =		param.transpose(1 ,    2		).contiguous()
    elif checkpoint_version >= 2.0:
        # other versions store [num_heads * num_splits * hidden_size, :]
        snake_case							:		Tuple        =		(num_heads, num_splits, hidden_size) + input_shape[1:]
        snake_case							:		Union[str, Any]        =		param.view(*__A		)
        snake_case							:		Dict        =		param.transpose(0 ,    1		).contiguous()
    snake_case							:		Any        =		param.view(*__A		)
    return param







def       lowercase					(	__A :					str ,    __A :					Dict ,    __A :					str		)				->			List[Any]:





    '''simple docstring'''




    snake_case							:		Dict        =		{}

    # old versions did not store training args
    snake_case							:		int        =		input_state_dict.get("""args""" ,    __A		)
    if ds_args is not None:
        # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
        # from pprint import pprint
        # pprint(vars(ds_args))

        snake_case							:		Dict        =		ds_args.padded_vocab_size
        snake_case							:		Dict        =		ds_args.max_position_embeddings
        snake_case							:		Optional[Any]        =		ds_args.hidden_size
        snake_case							:		List[str]        =		ds_args.num_layers
        snake_case							:		str        =		ds_args.num_attention_heads
        snake_case							:		Optional[Any]        =		ds_args.ffn_hidden_size
        # pprint(config)

    # The number of heads.
    snake_case							:		List[str]        =		config.n_head
    # The hidden_size per head.
    snake_case							:		List[Any]        =		config.n_embd // config.n_head
    # Megatron-LM checkpoint version
    if "checkpoint_version" in input_state_dict.keys():
        snake_case							:		List[Any]        =		input_state_dict["""checkpoint_version"""]
    else:
        snake_case							:		int        =		0.0

    # The model.
    snake_case							:		Optional[int]        =		input_state_dict["""model"""]
    # The language model.
    snake_case							:		Union[str, Any]        =		model["""language_model"""]
    # The embeddings.
    snake_case							:		List[str]        =		lm["""embedding"""]

    # The word embeddings.
    snake_case							:		Any        =		embeddings["""word_embeddings"""]["""weight"""]
    # Truncate the embedding table to vocab_size rows.
    snake_case							:		Union[str, Any]        =		word_embeddings[: config.vocab_size, :]
    snake_case							:		str        =		word_embeddings

    # The position embeddings.
    snake_case							:		Tuple        =		embeddings["""position_embeddings"""]["""weight"""]
    # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
    snake_case							:		Optional[Any]        =		pos_embeddings.size(0		)
    if n_positions != config.n_positions:
        raise ValueError(
            f"""pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"""		)
    # Store the position embeddings.
    snake_case							:		Tuple        =		pos_embeddings

    # The transformer.
    snake_case							:		Union[str, Any]        =		lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]

    # The regex to extract layer names.
    snake_case							:		Optional[int]        =		re.compile(r"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""		)

    # The simple map of names for "automated" rules.
    snake_case							:		str        =		{
        """attention.dense""": """.attn.c_proj.""",
        """self_attention.dense""": """.attn.c_proj.""",
        """mlp.dense_h_to_4h""": """.mlp.c_fc.""",
        """mlp.dense_4h_to_h""": """.mlp.c_proj.""",
    }

    # Extract the layers.
    for key, val in transformer.items():
        # Match the name.
        snake_case							:		Optional[Any]        =		layer_re.match(__A		)

        # Stop if that's not a layer
        if m is None:
            break

        # The index of the layer.
        snake_case							:		List[str]        =		int(m.group(1		)		)
        # The name of the operation.
        snake_case							:		Any        =		m.group(2		)
        # Is it a weight or a bias?
        snake_case							:		Any        =		m.group(3		)

        # The name of the layer.
        snake_case							:		Tuple        =		f"""transformer.h.{layer_idx}"""

        # For layernorm(s), simply store the layer norm.
        if op_name.endswith("""layernorm"""		):
            snake_case							:		Any        =		"""ln_1""" if op_name.startswith("""input"""		) else """ln_2"""
            snake_case							:		Any        =		val

        # Transpose the QKV matrix.
        elif (
            op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
        ) and weight_or_bias == "weight":
            # Insert a tensor of 1x1xDxD bias.
            snake_case							:		Any        =		torch.tril(torch.ones((n_positions, n_positions) ,    dtype=torch.floataa		)		).view(
                1 ,    1 ,    __A ,    __A		)
            snake_case							:		List[str]        =		causal_mask

            # Insert a "dummy" tensor for masked_bias.
            snake_case							:		Optional[int]        =		torch.tensor(-1E4 ,    dtype=torch.floataa		)
            snake_case							:		Any        =		masked_bias

            snake_case							:		Optional[Any]        =		fix_query_key_value_ordering(__A ,    __A ,    3 ,    __A ,    __A		)
            # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
            snake_case							:		int        =		out_val.transpose(0 ,    1		).contiguous()
            # Store.
            snake_case							:		List[Any]        =		out_val

        # Transpose the bias.
        elif (
            op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
        ) and weight_or_bias == "bias":
            snake_case							:		List[str]        =		fix_query_key_value_ordering(__A ,    __A ,    3 ,    __A ,    __A		)
            # Store. No change of shape.
            snake_case							:		List[str]        =		out_val

        # Transpose the weights.
        elif weight_or_bias == "weight":
            snake_case							:		str        =		megatron_to_transformers[op_name]
            snake_case							:		Tuple        =		val.transpose(0 ,    1		)

        # Copy the bias.
        elif weight_or_bias == "bias":
            snake_case							:		List[Any]        =		megatron_to_transformers[op_name]
            snake_case							:		int        =		val

    # DEBUG.
    assert config.n_layer == layer_idx + 1

    # The final layernorm.
    snake_case							:		List[Any]        =		transformer["""final_layernorm.weight"""]
    snake_case							:		Any        =		transformer["""final_layernorm.bias"""]

    # For LM head, transformers' wants the matrix to weight embeddings.
    snake_case							:		List[Any]        =		word_embeddings

    # It should be done!
    return output_state_dict







def       lowercase					(	)				->			Any:





    '''simple docstring'''




    snake_case							:		List[Any]        =		argparse.ArgumentParser()
    parser.add_argument("""--print-checkpoint-structure""" ,    action="""store_true"""		)
    parser.add_argument(
        """path_to_checkpoint""" ,    type=__A ,    help="""Path to the checkpoint file (.zip archive or direct .pt file)""" ,    )
    parser.add_argument(
        """--config_file""" ,    default="""""" ,    type=__A ,    help="""An optional config json file describing the pre-trained model.""" ,    )
    snake_case							:		List[Any]        =		parser.parse_args()

    # Extract the basename.
    snake_case							:		Tuple        =		os.path.dirname(args.path_to_checkpoint		)

    # Load the model.
    # the .zip is very optional, let's keep it for backward compatibility
    print(f"""Extracting PyTorch state dictionary from {args.path_to_checkpoint}"""		)
    if args.path_to_checkpoint.endswith(""".zip"""		):
        with zipfile.ZipFile(args.path_to_checkpoint ,    """r"""		) as checkpoint:
            with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""		) as pytorch_dict:
                snake_case							:		List[str]        =		torch.load(__A ,    map_location="""cpu"""		)
    else:
        snake_case							:		int        =		torch.load(args.path_to_checkpoint ,    map_location="""cpu"""		)

    snake_case							:		Dict        =		input_state_dict.get("""args""" ,    __A		)

    # Read the config, or default to the model released by NVIDIA.
    if args.config_file == "":
        if ds_args is not None:
            if ds_args.bias_gelu_fusion:
                snake_case							:		int        =		"""gelu_fast"""
            elif ds_args.openai_gelu:
                snake_case							:		Union[str, Any]        =		"""gelu_new"""
            else:
                snake_case							:		int        =		"""gelu"""
        else:
            # in the very early days this used to be "gelu_new"
            snake_case							:		str        =		"""gelu_new"""

        # Spell out all parameters in case the defaults change.
        snake_case							:		List[Any]        =		GPTaConfig(
            vocab_size=5_0257 ,    n_positions=1024 ,    n_embd=1024 ,    n_layer=24 ,    n_head=16 ,    n_inner=4096 ,    activation_function=__A ,    resid_pdrop=0.1 ,    embd_pdrop=0.1 ,    attn_pdrop=0.1 ,    layer_norm_epsilon=1E-5 ,    initializer_range=0.02 ,    summary_type="""cls_index""" ,    summary_use_proj=__A ,    summary_activation=__A ,    summary_proj_to_labels=__A ,    summary_first_dropout=0.1 ,    scale_attn_weights=__A ,    use_cache=__A ,    bos_token_id=5_0256 ,    eos_token_id=5_0256 ,    )
    else:
        snake_case							:		int        =		GPTaConfig.from_json_file(args.config_file		)

    snake_case							:		int        =		["""GPT2LMHeadModel"""]

    # Convert.
    print("""Converting"""		)
    snake_case							:		str        =		convert_megatron_checkpoint(__A ,    __A ,    __A		)

    # Print the structure of converted state dict.
    if args.print_checkpoint_structure:
        recursive_print(__A ,    __A		)

    # Add tokenizer class info to config
    # see https://github.com/huggingface/transformers/issues/13906)
    if ds_args is not None:
        snake_case							:		str        =		ds_args.tokenizer_type
        if tokenizer_type == "GPT2BPETokenizer":
            snake_case							:		Tuple        =		"""gpt2"""
        elif tokenizer_type == "PretrainedFromHF":
            snake_case							:		List[str]        =		ds_args.tokenizer_name_or_path
        else:
            raise ValueError(f"""Unrecognized tokenizer_type {tokenizer_type}"""		)
    else:
        snake_case							:		Any        =		"""gpt2"""

    snake_case							:		List[str]        =		AutoTokenizer.from_pretrained(__A		)
    snake_case							:		Optional[Any]        =		type(__A		).__name__
    snake_case							:		Tuple        =		tokenizer_class

    # Store the config to file.
    print("""Saving config"""		)
    config.save_pretrained(__A		)

    # Save tokenizer based on args
    print(f"""Adding {tokenizer_class} tokenizer files"""		)
    tokenizer.save_pretrained(__A		)

    # Store the state_dict to file.
    snake_case							:		List[str]        =		os.path.join(__A ,    """pytorch_model.bin"""		)
    print(f"""Saving checkpoint to \"{output_checkpoint_file}\""""		)
    torch.save(__A ,    __A		)


####################################################################################################

if __name__ == "__main__":
     main()

####################################################################################################







import argparse
import json
import os

import tensorstore as ts
import torch
from flax import serialization
from flax.traverse_util import flatten_dict, unflatten_dict
from tensorflow.io import gfile

from transformers.modeling_utils import dtype_byte_size
from transformers.models.switch_transformers.convert_switch_transformers_original_flax_checkpoint_to_pytorch import (
    rename_keys,
)
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME
from transformers.utils.hub import convert_file_size_to_int


def     lowercase	(    a     , a ):



		'''simple docstring'''




		if flax_key_tuple[-1] == "kernel" and flax_tensor.ndim == 3:
				# expert layer
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						flax_key_tuple[:-1] + ("weight",)
				SCREAMING_SNAKE_CASE_				:Any				     =						torch.permute(a     , (0, 2, 1) )
		elif flax_key_tuple[-1] == "kernel" and ".".join(a ):
				# linear layer
				SCREAMING_SNAKE_CASE_				:str				     =						flax_key_tuple[:-1] + ("weight",)
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						flax_tensor.T
		elif flax_key_tuple[-1] in ["scale", "embedding"]:
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						flax_key_tuple[:-1] + ("weight",)

		return flax_key_tuple, flax_tensor


def     lowercase	(    a     , a     , a ):



		'''simple docstring'''




		if "metadata" in layer:
				SCREAMING_SNAKE_CASE_				:Dict				     =						layer.split("metadata" )
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						"".join(split_layer[0] )[:-1]
				SCREAMING_SNAKE_CASE_				:str				     =						[tuple(("metadata" + split_layer[1]).split("/" ) )]
		elif "kvstore" in layer:
				SCREAMING_SNAKE_CASE_				:str				     =						layer.split("kvstore" )
				SCREAMING_SNAKE_CASE_				:str				     =						"".join(split_layer[0] )[:-1]
				SCREAMING_SNAKE_CASE_				:str				     =						[tuple(("kvstore" + split_layer[1]).split("/" ) )]

		else:
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						layer.split("/" )
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						"/".join(split_layer[:-1] )
				SCREAMING_SNAKE_CASE_				:int				     =						(split_layer[-1],)

		if "kvstore/path" in layer:
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						F"{switch_checkpoint_path}/{checkpoint_info[layer]}"
		elif "kvstore/driver" in layer:
				SCREAMING_SNAKE_CASE_				:Tuple				     =						"file"
		else:
				SCREAMING_SNAKE_CASE_				:str				     =						checkpoint_info[layer]

		return curr_real_layer_name, split_layer, content


def     lowercase	(    a     , a ):



		'''simple docstring'''




		SCREAMING_SNAKE_CASE_				:int				     =						rename_keys(a )
		SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						{}
		for k, v in current_block.items():
				SCREAMING_SNAKE_CASE_				:List[str]				     =						v
		SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						new_current_block
		torch.save(a     , a )


def     lowercase	(    a     , a     , a     , a     , a = WEIGHTS_NAME ):



		'''simple docstring'''




		SCREAMING_SNAKE_CASE_				:Optional[int]				     =						convert_file_size_to_int(a )
		SCREAMING_SNAKE_CASE_				:int				     =						[]
		SCREAMING_SNAKE_CASE_				:str				     =						{}
		SCREAMING_SNAKE_CASE_				:List[str]				     =						0
		SCREAMING_SNAKE_CASE_				:Optional[int]				     =						0

		os.makedirs(a     , exist_ok=a )
		with gfile.GFile(switch_checkpoint_path + "/checkpoint"     , "rb" ) as fp:
				SCREAMING_SNAKE_CASE_				:int				     =						serialization.msgpack_restore(fp.read() )["optimizer"]["target"]
				SCREAMING_SNAKE_CASE_				:Any				     =						flatten_dict(a     , sep="/" )

		SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						{}
		for layer in checkpoint_info.keys():
				SCREAMING_SNAKE_CASE_     ,				SCREAMING_SNAKE_CASE_     ,				SCREAMING_SNAKE_CASE_				:List[Any]				     =						get_key_and_tensorstore_dict(
				    a     , a     , a )
				if curr_real_layer_name in all_layers:
						SCREAMING_SNAKE_CASE_				:str				     =						content
				else:
						SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						{split_layer[-1]: content}

		for key in all_layers.keys():
				# open tensorstore file
				SCREAMING_SNAKE_CASE_				:Any				     =						ts.open(unflatten_dict(all_layers[key] ) ).result().read().result()
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						torch.tensor(a )
				SCREAMING_SNAKE_CASE_				:str				     =						raw_weights.numel() * dtype_byte_size(raw_weights.dtype )

				# use the renaming pattern from the small conversion scripts
				SCREAMING_SNAKE_CASE_     ,				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						rename_base_flax_keys(tuple(key.split("/" ) )     , a )
				SCREAMING_SNAKE_CASE_				:Any				     =						"/".join(a )

				# If this weight is going to tip up over the maximal size, we split.
				if current_block_size + weight_size > max_shard_size:
						SCREAMING_SNAKE_CASE_				:str				     =						os.path.join(
						    a     , weights_name.replace(".bin"     , F"-{len(a )+1:05d}-of-???.bin" ) )
						rename_and_save_block(a     , a )
						sharded_state_dicts.append(current_block.keys() )
						del current_block
						SCREAMING_SNAKE_CASE_				:Tuple				     =						{}
						SCREAMING_SNAKE_CASE_				:Dict				     =						0

				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						raw_weights.to(getattr(a     , a ) )
				current_block_size += weight_size
				total_size += weight_size

		# Add the last block
		SCREAMING_SNAKE_CASE_				:Dict				     =						os.path.join(a     , weights_name.replace(".bin"     , F"-{len(a )+1:05d}-of-???.bin" ) )
		rename_and_save_block(a     , a )
		sharded_state_dicts.append(current_block.keys() )

		# If we only have one shard, we return it
		if len(a ) == 1:
				return {weights_name: sharded_state_dicts[0]}, None

		# Otherwise, let's build the index
		SCREAMING_SNAKE_CASE_				:Optional[int]				     =						{}
		SCREAMING_SNAKE_CASE_				:int				     =						{}
		for idx, shard in enumerate(a ):
				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						weights_name.replace(
				    ".bin"     , F"-{idx+1:05d}-of-{len(a ):05d}.bin" )  # len(sharded_state_dicts):05d}
				SCREAMING_SNAKE_CASE_				:Any				     =						os.path.join(a     , weights_name.replace(".bin"     , F"-{idx+1:05d}-of-???.bin" ) )
				os.rename(a     , os.path.join(a     , a ) )
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						shard
				for key in shard:
						SCREAMING_SNAKE_CASE_				:str				     =						shard_file

    # Add the metadata
		SCREAMING_SNAKE_CASE_				:List[str]				     =						{"total_size": total_size}
		SCREAMING_SNAKE_CASE_				:Optional[int]				     =						{"metadata": metadata, "weight_map": weight_map}

		with open(os.path.join(a     , a )     , "w"     , encoding="utf-8" ) as f:
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						json.dumps(a     , indent=2     , sort_keys=a ) + "\n"
				f.write(a )

		return metadata, index


if __name__ == "__main__":
		SCREAMING_SNAKE_CASE__     					=			argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    "--switch_t5x_checkpoint_path",
		    default="/mnt/disks/disk_switch/original_checkpoints/switch-xxl-128/checkpoint_634600",
		    type=str,
		    required=False,
		    help="Path to a directory containing a folder per layer. Follows the original Google format.",
		)
		parser.add_argument("--max_shard_size", default="10GB", required=False, help="Max shard size")
		parser.add_argument("--dtype", default="bfloat16", type=str, required=False, help="dtype of the saved model")
		parser.add_argument(
		    "--pytorch_dump_folder_path",
		    default="/mnt/disks/disk_switch/original_checkpoints/switch-xxl-128-converted",
		    type=str,
		    required=False,
		    help="Path to the output pytorch model.",
		)
		SCREAMING_SNAKE_CASE__     					=			parser.parse_args()
		shard_on_the_fly(
		    args.switch_tax_checkpoint_path,
		    args.pytorch_dump_folder_path,
		    args.max_shard_size,
		    args.dtype,
		)


def     lowercase	(    ):



		'''simple docstring'''




		from transformers import SwitchTransformersConfig, SwitchTransformersForConditionalGeneration, TaTokenizer

		SCREAMING_SNAKE_CASE_				:Dict				     =						SwitchTransformersConfig.from_pretrained("google/switch-base-8" )
		config.save_pretrained("/home/arthur_huggingface_co/transformers/switch_converted" )
		SCREAMING_SNAKE_CASE_				:str				     =						SwitchTransformersForConditionalGeneration.from_pretrained(
		    "/home/arthur_huggingface_co/transformers/switch_converted"     , device_map="auto" )

		SCREAMING_SNAKE_CASE_				:List[Any]				     =						TaTokenizer.from_pretrained("t5-small" )
		SCREAMING_SNAKE_CASE_				:Optional[int]				     =						"A <extra_id_0> walks into a bar a orders a <extra_id_1> with <extra_id_2> pinch of <extra_id_3>."

		SCREAMING_SNAKE_CASE_				:List[Any]				     =						tokenizer(a     , return_tensors="pt" ).input_ids
		SCREAMING_SNAKE_CASE_				:List[str]				     =						model.generate(a     , decoder_start_token_id=0 )
		print(tokenizer.decode(out[0] ) )






