


import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig



def 	_lowerCAmelCase  (							lowerCAmelCase_							:List[Any]						,  lowerCAmelCase_							:str						,  lowerCAmelCase_							:List[str]=0							)->int:
   '''simple docstring'''






   if name is None:
      snake_case_    			=    None
   else:
      snake_case_    			=    """.""" * max(0						,  spaces - 2							) + """# {:""" + str(50 - spaces							) + """s}"""
      snake_case_    			=    fmt.format(UpperCamelCase__							)

   # Print and recurse (if needed).
   if isinstance(UpperCamelCase__						,  UpperCamelCase__							):
      if msg is not None:
         print(UpperCamelCase__							)
      for k in val.keys():
         recursive_print(UpperCamelCase__						,  val[k]						,  spaces + 2							)
   elif isinstance(UpperCamelCase__						,  torch.Tensor							):
      print(UpperCamelCase__						,  ":"						,  val.size()							)
   else:
      print(UpperCamelCase__						,  ":"						,  UpperCamelCase__							)



def 	_lowerCAmelCase  (							lowerCAmelCase_							:Optional[Any]						,  lowerCAmelCase_							:Any						,  lowerCAmelCase_							:List[str]						,  lowerCAmelCase_							:str						,  lowerCAmelCase_							:Optional[int]							)->List[Any]:
   '''simple docstring'''






   snake_case_    			=    param.size()
   if checkpoint_version == 1.0:
      # version 1.0 stores [num_heads * hidden_size * num_splits, :]
      snake_case_    			=    (num_heads, hidden_size, num_splits) + input_shape[1:]
      snake_case_    			=    param.view(*UpperCamelCase__							)
      snake_case_    			=    param.transpose(0						,  2							)
      snake_case_    			=    param.transpose(1						,  2							).contiguous()
   elif checkpoint_version >= 2.0:
      # other versions store [num_heads * num_splits * hidden_size, :]
      snake_case_    			=    (num_heads, num_splits, hidden_size) + input_shape[1:]
      snake_case_    			=    param.view(*UpperCamelCase__							)
      snake_case_    			=    param.transpose(0						,  1							).contiguous()
   snake_case_    			=    param.view(*UpperCamelCase__							)
   return param



def 	_lowerCAmelCase  (							lowerCAmelCase_							:Dict						,  lowerCAmelCase_							:str						,  lowerCAmelCase_							:Any							)->Union[str, Any]:
   '''simple docstring'''






   snake_case_    			=    {}

   # old versions did not store training args
   snake_case_    			=    input_state_dict.get("args"						,  UpperCamelCase__							)
   if ds_args is not None:
      # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
      # from pprint import pprint
      # pprint(vars(ds_args))

      snake_case_    			=    ds_args.padded_vocab_size
      snake_case_    			=    ds_args.max_position_embeddings
      snake_case_    			=    ds_args.hidden_size
      snake_case_    			=    ds_args.num_layers
      snake_case_    			=    ds_args.num_attention_heads
      snake_case_    			=    ds_args.ffn_hidden_size
      # pprint(config)

   # The number of heads.
   snake_case_    			=    config.n_head
   # The hidden_size per head.
   snake_case_    			=    config.n_embd // config.n_head
   # Megatron-LM checkpoint version
   if "checkpoint_version" in input_state_dict.keys():
      snake_case_    			=    input_state_dict["""checkpoint_version"""]
   else:
      snake_case_    			=    0.0

   # The model.
   snake_case_    			=    input_state_dict["""model"""]
   # The language model.
   snake_case_    			=    model["""language_model"""]
   # The embeddings.
   snake_case_    			=    lm["""embedding"""]

   # The word embeddings.
   snake_case_    			=    embeddings["""word_embeddings"""]["""weight"""]
   # Truncate the embedding table to vocab_size rows.
   snake_case_    			=    word_embeddings[: config.vocab_size, :]
   snake_case_    			=    word_embeddings

   # The position embeddings.
   snake_case_    			=    embeddings["""position_embeddings"""]["""weight"""]
   # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
   snake_case_    			=    pos_embeddings.size(0							)
   if n_positions != config.n_positions:
      raise ValueError(
          F'''pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'''							)
   # Store the position embeddings.
   snake_case_    			=    pos_embeddings

   # The transformer.
   snake_case_    			=    lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]

   # The regex to extract layer names.
   snake_case_    			=    re.compile(r"layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"							)

   # The simple map of names for "automated" rules.
   snake_case_    			=    {
       """attention.dense""": """.attn.c_proj.""",
       """self_attention.dense""": """.attn.c_proj.""",
       """mlp.dense_h_to_4h""": """.mlp.c_fc.""",
       """mlp.dense_4h_to_h""": """.mlp.c_proj.""",
   }

   # Extract the layers.
   for key, val in transformer.items():
      # Match the name.
      snake_case_    			=    layer_re.match(UpperCamelCase__							)

      # Stop if that's not a layer
      if m is None:
         break

      # The index of the layer.
      snake_case_    			=    int(m.group(1							)							)
      # The name of the operation.
      snake_case_    			=    m.group(2							)
      # Is it a weight or a bias?
      snake_case_    			=    m.group(3							)

      # The name of the layer.
      snake_case_    			=    F'''transformer.h.{layer_idx}'''

      # For layernorm(s), simply store the layer norm.
      if op_name.endswith("layernorm"							):
         snake_case_    			=    """ln_1""" if op_name.startswith("input"							) else """ln_2"""
         snake_case_    			=    val

      # Transpose the QKV matrix.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "weight":
         # Insert a tensor of 1x1xDxD bias.
         snake_case_    			=    torch.tril(torch.ones((n_positions, n_positions)						,  dtype=torch.floataa							)							).view(
             1						,  1						,  UpperCamelCase__						,  UpperCamelCase__							)
         snake_case_    			=    causal_mask

         # Insert a "dummy" tensor for masked_bias.
         snake_case_    			=    torch.tensor(-1e4						,  dtype=torch.floataa							)
         snake_case_    			=    masked_bias

         snake_case_    			=    fix_query_key_value_ordering(UpperCamelCase__						,  UpperCamelCase__						,  3						,  UpperCamelCase__						,  UpperCamelCase__							)
         # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
         snake_case_    			=    out_val.transpose(0						,  1							).contiguous()
         # Store.
         snake_case_    			=    out_val

      # Transpose the bias.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "bias":
         snake_case_    			=    fix_query_key_value_ordering(UpperCamelCase__						,  UpperCamelCase__						,  3						,  UpperCamelCase__						,  UpperCamelCase__							)
         # Store. No change of shape.
         snake_case_    			=    out_val

      # Transpose the weights.
      elif weight_or_bias == "weight":
         snake_case_    			=    megatron_to_transformers[op_name]
         snake_case_    			=    val.transpose(0						,  1							)

      # Copy the bias.
      elif weight_or_bias == "bias":
         snake_case_    			=    megatron_to_transformers[op_name]
         snake_case_    			=    val

      # DEBUG.
   assert config.n_layer == layer_idx + 1

   # The final layernorm.
   snake_case_    			=    transformer["""final_layernorm.weight"""]
   snake_case_    			=    transformer["""final_layernorm.bias"""]

   # For LM head, transformers' wants the matrix to weight embeddings.
   snake_case_    			=    word_embeddings

   # It should be done!
   return output_state_dict



def 	_lowerCAmelCase  (							)->Tuple:
   '''simple docstring'''






   snake_case_    			=    argparse.ArgumentParser()
   parser.add_argument("--print-checkpoint-structure"						,  action="store_true"							)
   parser.add_argument(
       "path_to_checkpoint"						,  type=UpperCamelCase__						,  help="Path to the checkpoint file (.zip archive or direct .pt file)"						,  )
   parser.add_argument(
       "--config_file"						,  default=""						,  type=UpperCamelCase__						,  help="An optional config json file describing the pre-trained model."						,  )
   snake_case_    			=    parser.parse_args()

   # Extract the basename.
   snake_case_    			=    os.path.dirname(args.path_to_checkpoint							)

   # Load the model.
   # the .zip is very optional, let's keep it for backward compatibility
   print(F'''Extracting PyTorch state dictionary from {args.path_to_checkpoint}'''							)
   if args.path_to_checkpoint.endswith(".zip"							):
      with zipfile.ZipFile(args.path_to_checkpoint						,  "r"							) as checkpoint:
         with checkpoint.open("release/mp_rank_00/model_optim_rng.pt"							) as pytorch_dict:
            snake_case_    			=    torch.load(UpperCamelCase__						,  map_location="cpu"							)
   else:
      snake_case_    			=    torch.load(args.path_to_checkpoint						,  map_location="cpu"							)

   snake_case_    			=    input_state_dict.get("args"						,  UpperCamelCase__							)

   # Read the config, or default to the model released by NVIDIA.
   if args.config_file == "":
      if ds_args is not None:
         if ds_args.bias_gelu_fusion:
            snake_case_    			=    """gelu_fast"""
         elif ds_args.openai_gelu:
            snake_case_    			=    """gelu_new"""
         else:
            snake_case_    			=    """gelu"""
      else:
         # in the very early days this used to be "gelu_new"
         snake_case_    			=    """gelu_new"""

      # Spell out all parameters in case the defaults change.
      snake_case_    			=    GPTaConfig(
          vocab_size=50_257						,  n_positions=1_024						,  n_embd=1_024						,  n_layer=24						,  n_head=16						,  n_inner=4_096						,  activation_function=UpperCamelCase__						,  resid_pdrop=0.1						,  embd_pdrop=0.1						,  attn_pdrop=0.1						,  layer_norm_epsilon=1e-5						,  initializer_range=0.0_2						,  summary_type="cls_index"						,  summary_use_proj=UpperCamelCase__						,  summary_activation=UpperCamelCase__						,  summary_proj_to_labels=UpperCamelCase__						,  summary_first_dropout=0.1						,  scale_attn_weights=UpperCamelCase__						,  use_cache=UpperCamelCase__						,  bos_token_id=50_256						,  eos_token_id=50_256						,  )
   else:
      snake_case_    			=    GPTaConfig.from_json_file(args.config_file							)

   snake_case_    			=    ["""GPT2LMHeadModel"""]

   # Convert.
   print("Converting"							)
   snake_case_    			=    convert_megatron_checkpoint(UpperCamelCase__						,  UpperCamelCase__						,  UpperCamelCase__							)

   # Print the structure of converted state dict.
   if args.print_checkpoint_structure:
      recursive_print(UpperCamelCase__						,  UpperCamelCase__							)

   # Add tokenizer class info to config
   # see https://github.com/huggingface/transformers/issues/13906)
   if ds_args is not None:
      snake_case_    			=    ds_args.tokenizer_type
      if tokenizer_type == "GPT2BPETokenizer":
         snake_case_    			=    """gpt2"""
      elif tokenizer_type == "PretrainedFromHF":
         snake_case_    			=    ds_args.tokenizer_name_or_path
      else:
         raise ValueError(F'''Unrecognized tokenizer_type {tokenizer_type}'''							)
   else:
      snake_case_    			=    """gpt2"""

   snake_case_    			=    AutoTokenizer.from_pretrained(UpperCamelCase__							)
   snake_case_    			=    type(UpperCamelCase__							).__name__
   snake_case_    			=    tokenizer_class

   # Store the config to file.
   print("Saving config"							)
   config.save_pretrained(UpperCamelCase__							)

   # Save tokenizer based on args
   print(F'''Adding {tokenizer_class} tokenizer files'''							)
   tokenizer.save_pretrained(UpperCamelCase__							)

   # Store the state_dict to file.
   snake_case_    			=    os.path.join(UpperCamelCase__						,  "pytorch_model.bin"							)
   print(F'''Saving checkpoint to \"{output_checkpoint_file}\"'''							)
   torch.save(UpperCamelCase__						,  UpperCamelCase__							)


####################################################################################################

if __name__ == "__main__":
   main()

####################################################################################################












"""simple docstring"""




import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case          =   logging.get_logger(__name__)

_snake_case          =   {
    'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/config.json',
}






class        UpperCamelCase			(	snake_case_      ):
 UpperCamelCase					:       int			=					'''mvp'''
 UpperCamelCase					:       Union[str, Any]			=					['''past_key_values''']
 UpperCamelCase					:       Any			=					{'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}


 def __init__(     self		:						List[str]							,						UpperCAmelCase__		:						List[str]=50267							,						UpperCAmelCase__		:						Optional[Any]=1024							,						UpperCAmelCase__		:						Tuple=12							,						UpperCAmelCase__		:						Optional[Any]=4096							,						UpperCAmelCase__		:						int=16							,						UpperCAmelCase__		:						Tuple=12							,						UpperCAmelCase__		:						int=4096							,						UpperCAmelCase__		:						List[Any]=16							,						UpperCAmelCase__		:						Tuple=0.0							,						UpperCAmelCase__		:						Tuple=0.0							,						UpperCAmelCase__		:						Tuple="gelu"							,						UpperCAmelCase__		:						Union[str, Any]=1024							,						UpperCAmelCase__		:						List[str]=0.1							,						UpperCAmelCase__		:						Any=0.0							,						UpperCAmelCase__		:						Dict=0.0							,						UpperCAmelCase__		:						Tuple=0.0_2							,						UpperCAmelCase__		:						Tuple=0.0							,						UpperCAmelCase__		:						Optional[Any]=False							,						UpperCAmelCase__		:						int=True							,						UpperCAmelCase__		:						Tuple=1							,						UpperCAmelCase__		:						Dict=0							,						UpperCAmelCase__		:						Union[str, Any]=2							,						UpperCAmelCase__		:						Optional[int]=True							,						UpperCAmelCase__		:						Tuple=2							,						UpperCAmelCase__		:						Any=2							,						UpperCAmelCase__		:						Optional[Any]=False							,						UpperCAmelCase__		:						Dict=100							,						UpperCAmelCase__		:						Union[str, Any]=800							,						**UpperCAmelCase__		:						Dict							,						)					->       List[Any]:
   _a  :       Any             =	vocab_size
   _a  :       Any             =	max_position_embeddings
   _a  :       Union[str, Any]             =	d_model
   _a  :       List[str]             =	encoder_ffn_dim
   _a  :       List[Any]             =	encoder_layers
   _a  :       Dict             =	encoder_attention_heads
   _a  :       Tuple             =	decoder_ffn_dim
   _a  :       List[Any]             =	decoder_layers
   _a  :       Optional[Any]             =	decoder_attention_heads
   _a  :       Optional[Any]             =	dropout
   _a  :       str             =	attention_dropout
   _a  :       Dict             =	activation_dropout
   _a  :       Any             =	activation_function
   _a  :       Tuple             =	init_std
   _a  :       Dict             =	encoder_layerdrop
   _a  :       Optional[int]             =	decoder_layerdrop
   _a  :       Optional[Any]             =	classifier_dropout
   _a  :       List[Any]             =	use_cache
   _a  :       Dict             =	encoder_layers
   _a  :       str             =	scale_embedding  # scale factor will be sqrt(d_model) if True
   _a  :       int             =	use_prompt
   _a  :       Dict             =	prompt_length
   _a  :       Dict             =	prompt_mid_dim

   super().__init__(
       pad_token_id=UpperCAmelCase__							,						bos_token_id=UpperCAmelCase__							,						eos_token_id=UpperCAmelCase__							,						is_encoder_decoder=UpperCAmelCase__							,						decoder_start_token_id=UpperCAmelCase__							,						forced_eos_token_id=UpperCAmelCase__							,						**UpperCAmelCase__							,						)

   if self.forced_bos_token_id is None and kwargs.get("""force_bos_token_to_be_generated"""							,						UpperCAmelCase__							):
     _a  :       List[str]             =	self.bos_token_id
     warnings.warn(
         f"""Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. """
         """The config can simply be saved and uploaded again to be fixed."""							)

