





# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig

def   a_    ( lowerCAmelCase_      :			Tuple,	lowerCAmelCase_      :			Tuple,	lowerCAmelCase_      :			Dict=0	):
   # Format the message.
   if name is None:
      __lowerCAmelCase								=	None
   else:
      __lowerCAmelCase								=	'.' * max(0,	spaces - 2	) + '# {:' + str(50 - spaces	) + 's}'
      __lowerCAmelCase								=	fmt.format(lowerCAmelCase_	)

   # Print and recurse (if needed).
   if isinstance(lowerCAmelCase_,	lowerCAmelCase_	):
      if msg is not None:
         print(lowerCAmelCase_	)
      for k in val.keys():
         recursive_print(lowerCAmelCase_,	val[k],	spaces + 2	)
   elif isinstance(lowerCAmelCase_,	torch.Tensor	):
      print(lowerCAmelCase_,	':',	val.size()	)
   else:
      print(lowerCAmelCase_,	':',	lowerCAmelCase_	)

def   a_    ( lowerCAmelCase_      :			List[Any],	lowerCAmelCase_      :			Any,	lowerCAmelCase_      :			Dict,	lowerCAmelCase_      :			Dict,	lowerCAmelCase_      :			Tuple	):
   # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
   # for compatibility with later versions of NVIDIA Megatron-LM.
   # The inverse operation is performed inside Megatron-LM to read checkpoints:
   # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
   # If param is the weight tensor of the self-attention block, the returned tensor
   # will have to be transposed one more time to be read by HuggingFace GPT2.
   __lowerCAmelCase								=	param.size()
   if checkpoint_version == 1.0:
      # version 1.0 stores [num_heads * hidden_size * num_splits, :]
      __lowerCAmelCase								=	(num_heads, hidden_size, num_splits) + input_shape[1:]
      __lowerCAmelCase								=	param.view(*lowerCAmelCase_	)
      __lowerCAmelCase								=	param.transpose(0,	2	)
      __lowerCAmelCase								=	param.transpose(1,	2	).contiguous()
   elif checkpoint_version >= 2.0:
      # other versions store [num_heads * num_splits * hidden_size, :]
      __lowerCAmelCase								=	(num_heads, num_splits, hidden_size) + input_shape[1:]
      __lowerCAmelCase								=	param.view(*lowerCAmelCase_	)
      __lowerCAmelCase								=	param.transpose(0,	1	).contiguous()
   __lowerCAmelCase								=	param.view(*lowerCAmelCase_	)
   return param

def   a_    ( lowerCAmelCase_      :			Optional[Any],	lowerCAmelCase_      :			Union[str, Any],	lowerCAmelCase_      :			List[str]	):
   # The converted output model.
   __lowerCAmelCase								=	{}

   # old versions did not store training args
   __lowerCAmelCase								=	input_state_dict.get('args',	lowerCAmelCase_	)
   if ds_args is not None:
      # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
      # from pprint import pprint
      # pprint(vars(ds_args))

      __lowerCAmelCase								=	ds_args.padded_vocab_size
      __lowerCAmelCase								=	ds_args.max_position_embeddings
      __lowerCAmelCase								=	ds_args.hidden_size
      __lowerCAmelCase								=	ds_args.num_layers
      __lowerCAmelCase								=	ds_args.num_attention_heads
      __lowerCAmelCase								=	ds_args.ffn_hidden_size
      # pprint(config)

   # The number of heads.
   __lowerCAmelCase								=	config.n_head
   # The hidden_size per head.
   __lowerCAmelCase								=	config.n_embd // config.n_head
   # Megatron-LM checkpoint version
   if "checkpoint_version" in input_state_dict.keys():
      __lowerCAmelCase								=	input_state_dict['checkpoint_version']
   else:
      __lowerCAmelCase								=	0.0

   # The model.
   __lowerCAmelCase								=	input_state_dict['model']
   # The language model.
   __lowerCAmelCase								=	model['language_model']
   # The embeddings.
   __lowerCAmelCase								=	lm['embedding']

   # The word embeddings.
   __lowerCAmelCase								=	embeddings['word_embeddings']['weight']
   # Truncate the embedding table to vocab_size rows.
   __lowerCAmelCase								=	word_embeddings[: config.vocab_size, :]
   __lowerCAmelCase								=	word_embeddings

   # The position embeddings.
   __lowerCAmelCase								=	embeddings['position_embeddings']['weight']
   # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
   __lowerCAmelCase								=	pos_embeddings.size(0	)
   if n_positions != config.n_positions:
      raise ValueError(
          F"""pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"""	)
   # Store the position embeddings.
   __lowerCAmelCase								=	pos_embeddings

   # The transformer.
   __lowerCAmelCase								=	lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

   # The regex to extract layer names.
   __lowerCAmelCase								=	re.compile(R'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'	)

   # The simple map of names for "automated" rules.
   __lowerCAmelCase								=	{
       'attention.dense': '.attn.c_proj.',
       'self_attention.dense': '.attn.c_proj.',
       'mlp.dense_h_to_4h': '.mlp.c_fc.',
       'mlp.dense_4h_to_h': '.mlp.c_proj.',
   }

   # Extract the layers.
   for key, val in transformer.items():
      # Match the name.
      __lowerCAmelCase								=	layer_re.match(lowerCAmelCase_	)

      # Stop if that's not a layer
      if m is None:
         break

      # The index of the layer.
      __lowerCAmelCase								=	int(m.group(1	)	)
      # The name of the operation.
      __lowerCAmelCase								=	m.group(2	)
      # Is it a weight or a bias?
      __lowerCAmelCase								=	m.group(3	)

      # The name of the layer.
      __lowerCAmelCase								=	F"""transformer.h.{layer_idx}"""

      # For layernorm(s), simply store the layer norm.
      if op_name.endswith('layernorm'	):
         __lowerCAmelCase								=	'ln_1' if op_name.startswith('input'	) else 'ln_2'
         __lowerCAmelCase								=	val

      # Transpose the QKV matrix.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "weight":
         # Insert a tensor of 1x1xDxD bias.
         __lowerCAmelCase								=	torch.tril(torch.ones((n_positions, n_positions),	dtype=torch.floataa	)	).view(
             1,	1,	lowerCAmelCase_,	lowerCAmelCase_	)
         __lowerCAmelCase								=	causal_mask

         # Insert a "dummy" tensor for masked_bias.
         __lowerCAmelCase								=	torch.tensor(-1E4,	dtype=torch.floataa	)
         __lowerCAmelCase								=	masked_bias

         __lowerCAmelCase								=	fix_query_key_value_ordering(lowerCAmelCase_,	lowerCAmelCase_,	3,	lowerCAmelCase_,	lowerCAmelCase_	)
         # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
         __lowerCAmelCase								=	out_val.transpose(0,	1	).contiguous()
         # Store.
         __lowerCAmelCase								=	out_val

      # Transpose the bias.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "bias":
         __lowerCAmelCase								=	fix_query_key_value_ordering(lowerCAmelCase_,	lowerCAmelCase_,	3,	lowerCAmelCase_,	lowerCAmelCase_	)
         # Store. No change of shape.
         __lowerCAmelCase								=	out_val

      # Transpose the weights.
      elif weight_or_bias == "weight":
         __lowerCAmelCase								=	megatron_to_transformers[op_name]
         __lowerCAmelCase								=	val.transpose(0,	1	)

      # Copy the bias.
      elif weight_or_bias == "bias":
         __lowerCAmelCase								=	megatron_to_transformers[op_name]
         __lowerCAmelCase								=	val

    # DEBUG.
   assert config.n_layer == layer_idx + 1

   # The final layernorm.
   __lowerCAmelCase								=	transformer['final_layernorm.weight']
   __lowerCAmelCase								=	transformer['final_layernorm.bias']

   # For LM head, transformers' wants the matrix to weight embeddings.
   __lowerCAmelCase								=	word_embeddings

   # It should be done!
   return output_state_dict

def   a_    ( ):
   # Create the argument parser.
   __lowerCAmelCase								=	argparse.ArgumentParser()
   parser.add_argument('--print-checkpoint-structure',	action='store_true'	)
   parser.add_argument(
       'path_to_checkpoint',	type=lowerCAmelCase_,	help='Path to the checkpoint file (.zip archive or direct .pt file)',	)
   parser.add_argument(
       '--config_file',	default='',	type=lowerCAmelCase_,	help='An optional config json file describing the pre-trained model.',	)
   __lowerCAmelCase								=	parser.parse_args()

   # Extract the basename.
   __lowerCAmelCase								=	os.path.dirname(args.path_to_checkpoint	)

   # Load the model.
   # the .zip is very optional, let's keep it for backward compatibility
   print(F"""Extracting PyTorch state dictionary from {args.path_to_checkpoint}"""	)
   if args.path_to_checkpoint.endswith('.zip'	):
      with zipfile.ZipFile(args.path_to_checkpoint,	'r'	) as checkpoint:
         with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'	) as pytorch_dict:
            __lowerCAmelCase								=	torch.load(lowerCAmelCase_,	map_location='cpu'	)
   else:
      __lowerCAmelCase								=	torch.load(args.path_to_checkpoint,	map_location='cpu'	)

   __lowerCAmelCase								=	input_state_dict.get('args',	lowerCAmelCase_	)

   # Read the config, or default to the model released by NVIDIA.
   if args.config_file == "":
      if ds_args is not None:
         if ds_args.bias_gelu_fusion:
            __lowerCAmelCase								=	'gelu_fast'
         elif ds_args.openai_gelu:
            __lowerCAmelCase								=	'gelu_new'
         else:
            __lowerCAmelCase								=	'gelu'
      else:
         # in the very early days this used to be "gelu_new"
         __lowerCAmelCase								=	'gelu_new'

      # Spell out all parameters in case the defaults change.
      __lowerCAmelCase								=	GPTaConfig(
          vocab_size=5_0257,	n_positions=1024,	n_embd=1024,	n_layer=24,	n_head=16,	n_inner=4096,	activation_function=lowerCAmelCase_,	resid_pdrop=0.1,	embd_pdrop=0.1,	attn_pdrop=0.1,	layer_norm_epsilon=1E-5,	initializer_range=0.02,	summary_type='cls_index',	summary_use_proj=lowerCAmelCase_,	summary_activation=lowerCAmelCase_,	summary_proj_to_labels=lowerCAmelCase_,	summary_first_dropout=0.1,	scale_attn_weights=lowerCAmelCase_,	use_cache=lowerCAmelCase_,	bos_token_id=5_0256,	eos_token_id=5_0256,	)
   else:
      __lowerCAmelCase								=	GPTaConfig.from_json_file(args.config_file	)

   __lowerCAmelCase								=	['GPT2LMHeadModel']

   # Convert.
   print('Converting'	)
   __lowerCAmelCase								=	convert_megatron_checkpoint(lowerCAmelCase_,	lowerCAmelCase_,	lowerCAmelCase_	)

   # Print the structure of converted state dict.
   if args.print_checkpoint_structure:
      recursive_print(lowerCAmelCase_,	lowerCAmelCase_	)

   # Add tokenizer class info to config
   # see https://github.com/huggingface/transformers/issues/13906)
   if ds_args is not None:
      __lowerCAmelCase								=	ds_args.tokenizer_type
      if tokenizer_type == "GPT2BPETokenizer":
         __lowerCAmelCase								=	'gpt2'
      elif tokenizer_type == "PretrainedFromHF":
         __lowerCAmelCase								=	ds_args.tokenizer_name_or_path
      else:
         raise ValueError(F"""Unrecognized tokenizer_type {tokenizer_type}"""	)
   else:
      __lowerCAmelCase								=	'gpt2'

   __lowerCAmelCase								=	AutoTokenizer.from_pretrained(lowerCAmelCase_	)
   __lowerCAmelCase								=	type(lowerCAmelCase_	).__name__
   __lowerCAmelCase								=	tokenizer_class

   # Store the config to file.
   print('Saving config'	)
   config.save_pretrained(lowerCAmelCase_	)

   # Save tokenizer based on args
   print(F"""Adding {tokenizer_class} tokenizer files"""	)
   tokenizer.save_pretrained(lowerCAmelCase_	)

   # Store the state_dict to file.
   __lowerCAmelCase								=	os.path.join(lowerCAmelCase_,	'pytorch_model.bin'	)
   print(F"""Saving checkpoint to \"{output_checkpoint_file}\""""	)
   torch.save(lowerCAmelCase_,	lowerCAmelCase_	)


####################################################################################################

if __name__ == "__main__":
     main()

####################################################################################################








import math_equivalence  # From: git+https://github.com/hendrycks/math.git

import datasets


_snake_case  :    Tuple						    =  '\\n@article{hendrycksmath2021,\n  title={Measuring Mathematical Problem Solving With the MATH Dataset},\n  author={Dan Hendrycks\n    and Collin Burns\n    and Saurav Kadavath\n    and Akul Arora\n    and Steven Basart\n    and Eric Tang\n    and Dawn Song\n    and Jacob Steinhardt},\n  journal={arXiv preprint arXiv:2103.03874},\n  year={2021}\n}\n'


_snake_case  :    str						    =  '\\nThis metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset.\nIt first canonicalizes the inputs (e.g., converting "1/2" to "\\frac{1}{2}") and then computes accuracy.\n'


_snake_case  :    List[str]						    =  R'\nCalculates accuracy after canonicalizing inputs.\n\nArgs:\n    predictions: list of predictions to score. Each prediction\n        is a string that contains natural language and LaTex.\n    references: list of reference for each prediction. Each\n        reference is a string that contains natural language\n        and LaTex.\nReturns:\n    accuracy: accuracy after canonicalizing inputs\n        (e.g., converting "1/2" to "\\frac{1}{2}")\n\nExamples:\n    >>> metric = datasets.load_metric("competition_math")\n    >>> results = metric.compute(references=["\\frac{1}{2}"], predictions=["1/2"])\n    >>> print(results)\n    {\'accuracy\': 1.0}\n'



@datasets.utils.file_utils.add_end_docstrings(_DESCRIPTION		,    _KWARGS_DESCRIPTION    )
class 				_UpperCAmelCase    (    datasets.Metric    ):

  """simple docstring"""


  def 				lowercase		(				self :		str     )							->						int:
     return datasets.MetricInfo(
         description=_DESCRIPTION					,  citation=_CITATION					,  inputs_description=_KWARGS_DESCRIPTION					,  features=datasets.Features(
             {
                 'predictions': datasets.Value('string'     ),
                 'references': datasets.Value('string'     ),
             }     )					,  homepage='https://github.com/hendrycks/math'					,  codebase_urls=['https://github.com/hendrycks/math']					,  )






  def 				lowercase		(				self :		str					,  lowerCAmelCase_ :		Any					,  lowerCAmelCase_ :		Optional[int]     )							->						List[Any]:
     __lowerCAmelCase								=	0.0
     for i, j in zip(lowerCAmelCase_					,  lowerCAmelCase_     ):
        n_correct += 1.0 if math_equivalence.is_equiv(lowerCAmelCase_					,  lowerCAmelCase_     ) else 0.0
     __lowerCAmelCase								=	n_correct / len(lowerCAmelCase_     )
     return {
         "accuracy": accuracy,
     }

