





# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig







def    __lowerCamelCase					(							__a   :str    ,     __a   :str    ,     __a   :Union[str, Any]=0    )		->				str:






		"""simple docstring"""

		if name is None:
				A__        =   None
		else:
				A__        =   """.""" * max(0    ,     spaces - 2    ) + """# {:""" + str(5_0 - spaces    ) + """s}"""
				A__        =   fmt.format(__a    )

		# Print and recurse (if needed).
		if isinstance(__a    ,     __a    ):
				if msg is not None:
						print(__a    )
				for k in val.keys():
						recursive_print(__a    ,     val[k]    ,     spaces + 2    )
		elif isinstance(__a    ,     torch.Tensor    ):
				print(__a    ,     """:"""    ,     val.size()    )
		else:
				print(__a    ,     """:"""    ,     __a    )







def    __lowerCamelCase					(							__a   :Optional[int]    ,     __a   :Tuple    ,     __a   :Optional[Any]    ,     __a   :Tuple    ,     __a   :Optional[int]    )		->				List[Any]:






		"""simple docstring"""

		A__        =   param.size()
		if checkpoint_version == 1.0:
				# version 1.0 stores [num_heads * hidden_size * num_splits, :]
				A__        =   (num_heads, hidden_size, num_splits) + input_shape[1:]
				A__        =   param.view(*__a    )
				A__        =   param.transpose(0    ,     2    )
				A__        =   param.transpose(1    ,     2    ).contiguous()
		elif checkpoint_version >= 2.0:
				# other versions store [num_heads * num_splits * hidden_size, :]
				A__        =   (num_heads, num_splits, hidden_size) + input_shape[1:]
				A__        =   param.view(*__a    )
				A__        =   param.transpose(0    ,     1    ).contiguous()
		A__        =   param.view(*__a    )
		return param







def    __lowerCamelCase					(							__a   :Tuple    ,     __a   :str    ,     __a   :Tuple    )		->				Union[str, Any]:






		"""simple docstring"""

		A__        =   {}

		# old versions did not store training args
		A__        =   input_state_dict.get("""args"""    ,     __a    )
		if ds_args is not None:
				# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
				# from pprint import pprint
				# pprint(vars(ds_args))

				A__        =   ds_args.padded_vocab_size
				A__        =   ds_args.max_position_embeddings
				A__        =   ds_args.hidden_size
				A__        =   ds_args.num_layers
				A__        =   ds_args.num_attention_heads
				A__        =   ds_args.ffn_hidden_size
				# pprint(config)

		# The number of heads.
		A__        =   config.n_head
		# The hidden_size per head.
		A__        =   config.n_embd // config.n_head
		# Megatron-LM checkpoint version
		if "checkpoint_version" in input_state_dict.keys():
				A__        =   input_state_dict["""checkpoint_version"""]
		else:
				A__        =   0.0

		# The model.
		A__        =   input_state_dict["""model"""]
		# The language model.
		A__        =   model["""language_model"""]
		# The embeddings.
		A__        =   lm["""embedding"""]

		# The word embeddings.
		A__        =   embeddings["""word_embeddings"""]["""weight"""]
		# Truncate the embedding table to vocab_size rows.
		A__        =   word_embeddings[: config.vocab_size, :]
		A__        =   word_embeddings

		# The position embeddings.
		A__        =   embeddings["""position_embeddings"""]["""weight"""]
		# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
		A__        =   pos_embeddings.size(0    )
		if n_positions != config.n_positions:
				raise ValueError(
				    F'pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'    )
		# Store the position embeddings.
		A__        =   pos_embeddings

		# The transformer.
		A__        =   lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]

		# The regex to extract layer names.
		A__        =   re.compile(R"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""    )

		# The simple map of names for "automated" rules.
		A__        =   {
		    """attention.dense""": """.attn.c_proj.""",
		    """self_attention.dense""": """.attn.c_proj.""",
		    """mlp.dense_h_to_4h""": """.mlp.c_fc.""",
		    """mlp.dense_4h_to_h""": """.mlp.c_proj.""",
		}

		# Extract the layers.
		for key, val in transformer.items():
				# Match the name.
				A__        =   layer_re.match(__a    )

				# Stop if that's not a layer
				if m is None:
						break

				# The index of the layer.
				A__        =   int(m.group(1    )    )
				# The name of the operation.
				A__        =   m.group(2    )
				# Is it a weight or a bias?
				A__        =   m.group(3    )

				# The name of the layer.
				A__        =   F'transformer.h.{layer_idx}'

				# For layernorm(s), simply store the layer norm.
				if op_name.endswith("""layernorm"""    ):
						A__        =   """ln_1""" if op_name.startswith("""input"""    ) else """ln_2"""
						A__        =   val

				# Transpose the QKV matrix.
				elif (
				    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
				) and weight_or_bias == "weight":
						# Insert a tensor of 1x1xDxD bias.
						A__        =   torch.tril(torch.ones((n_positions, n_positions)    ,     dtype=torch.floataa    )    ).view(
						    1    ,     1    ,     __a    ,     __a    )
						A__        =   causal_mask

						# Insert a "dummy" tensor for masked_bias.
						A__        =   torch.tensor(-1E4    ,     dtype=torch.floataa    )
						A__        =   masked_bias

						A__        =   fix_query_key_value_ordering(__a    ,     __a    ,     3    ,     __a    ,     __a    )
						# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
						A__        =   out_val.transpose(0    ,     1    ).contiguous()
						# Store.
						A__        =   out_val

				# Transpose the bias.
				elif (
				    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
				) and weight_or_bias == "bias":
						A__        =   fix_query_key_value_ordering(__a    ,     __a    ,     3    ,     __a    ,     __a    )
						# Store. No change of shape.
						A__        =   out_val

				# Transpose the weights.
				elif weight_or_bias == "weight":
						A__        =   megatron_to_transformers[op_name]
						A__        =   val.transpose(0    ,     1    )

				# Copy the bias.
				elif weight_or_bias == "bias":
						A__        =   megatron_to_transformers[op_name]
						A__        =   val

    # DEBUG.
		assert config.n_layer == layer_idx + 1

		# The final layernorm.
		A__        =   transformer["""final_layernorm.weight"""]
		A__        =   transformer["""final_layernorm.bias"""]

		# For LM head, transformers' wants the matrix to weight embeddings.
		A__        =   word_embeddings

		# It should be done!
		return output_state_dict







def    __lowerCamelCase					(							)		->				Dict:






		"""simple docstring"""

		A__        =   argparse.ArgumentParser()
		parser.add_argument("""--print-checkpoint-structure"""    ,     action="""store_true"""    )
		parser.add_argument(
		    """path_to_checkpoint"""    ,     type=__a    ,     help="""Path to the checkpoint file (.zip archive or direct .pt file)"""    ,     )
		parser.add_argument(
		    """--config_file"""    ,     default=""""""    ,     type=__a    ,     help="""An optional config json file describing the pre-trained model."""    ,     )
		A__        =   parser.parse_args()

		# Extract the basename.
		A__        =   os.path.dirname(args.path_to_checkpoint    )

		# Load the model.
		# the .zip is very optional, let's keep it for backward compatibility
		print(F'Extracting PyTorch state dictionary from {args.path_to_checkpoint}'    )
		if args.path_to_checkpoint.endswith(""".zip"""    ):
				with zipfile.ZipFile(args.path_to_checkpoint    ,     """r"""    ) as checkpoint:
						with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""    ) as pytorch_dict:
								A__        =   torch.load(__a    ,     map_location="""cpu"""    )
		else:
				A__        =   torch.load(args.path_to_checkpoint    ,     map_location="""cpu"""    )

		A__        =   input_state_dict.get("""args"""    ,     __a    )

		# Read the config, or default to the model released by NVIDIA.
		if args.config_file == "":
				if ds_args is not None:
						if ds_args.bias_gelu_fusion:
								A__        =   """gelu_fast"""
						elif ds_args.openai_gelu:
								A__        =   """gelu_new"""
						else:
								A__        =   """gelu"""
				else:
						# in the very early days this used to be "gelu_new"
						A__        =   """gelu_new"""

				# Spell out all parameters in case the defaults change.
				A__        =   GPTaConfig(
				    vocab_size=5_0_2_5_7    ,     n_positions=1_0_2_4    ,     n_embd=1_0_2_4    ,     n_layer=2_4    ,     n_head=1_6    ,     n_inner=4_0_9_6    ,     activation_function=__a    ,     resid_pdrop=0.1    ,     embd_pdrop=0.1    ,     attn_pdrop=0.1    ,     layer_norm_epsilon=1E-5    ,     initializer_range=0.02    ,     summary_type="""cls_index"""    ,     summary_use_proj=__a    ,     summary_activation=__a    ,     summary_proj_to_labels=__a    ,     summary_first_dropout=0.1    ,     scale_attn_weights=__a    ,     use_cache=__a    ,     bos_token_id=5_0_2_5_6    ,     eos_token_id=5_0_2_5_6    ,     )
		else:
				A__        =   GPTaConfig.from_json_file(args.config_file    )

		A__        =   ["""GPT2LMHeadModel"""]

		# Convert.
		print("""Converting"""    )
		A__        =   convert_megatron_checkpoint(__a    ,     __a    ,     __a    )

		# Print the structure of converted state dict.
		if args.print_checkpoint_structure:
				recursive_print(__a    ,     __a    )

		# Add tokenizer class info to config
		# see https://github.com/huggingface/transformers/issues/13906)
		if ds_args is not None:
				A__        =   ds_args.tokenizer_type
				if tokenizer_type == "GPT2BPETokenizer":
						A__        =   """gpt2"""
				elif tokenizer_type == "PretrainedFromHF":
						A__        =   ds_args.tokenizer_name_or_path
				else:
						raise ValueError(F'Unrecognized tokenizer_type {tokenizer_type}'    )
		else:
				A__        =   """gpt2"""

		A__        =   AutoTokenizer.from_pretrained(__a    )
		A__        =   type(__a    ).__name__
		A__        =   tokenizer_class

		# Store the config to file.
		print("""Saving config"""    )
		config.save_pretrained(__a    )

		# Save tokenizer based on args
		print(F'Adding {tokenizer_class} tokenizer files'    )
		tokenizer.save_pretrained(__a    )

		# Store the state_dict to file.
		A__        =   os.path.join(__a    ,     """pytorch_model.bin"""    )
		print(F'Saving checkpoint to "{output_checkpoint_file}"'    )
		torch.save(__a    ,     __a    )


####################################################################################################

if __name__ == "__main__":
					main()

####################################################################################################










import argparse
from collections import defaultdict

import yaml


A    :					str =      '''docs/source/en/_toctree.yml'''







def    __lowerCamelCase					(							__a   :str    )		->				List[Any]:






		"""simple docstring"""

		A__        =   defaultdict(__a    )
		A__        =   []
		A__        =   []
		for doc in doc_list:
				if "local" in doc:
						counts[doc["local"]] += 1

				if doc["title"].lower() == "overview":
						overview_doc.append({"""local""": doc["""local"""], """title""": doc["""title"""]}    )
				else:
						new_doc_list.append(__a    )

		A__        =   new_doc_list
		A__        =   [key for key, value in counts.items() if value > 1]

		A__        =   []
		for duplicate_key in duplicates:
				A__        =   list({doc["""title"""] for doc in doc_list if doc["""local"""] == duplicate_key}    )
				if len(__a    ) > 1:
						raise ValueError(
						    F'{duplicate_key} is present several times in the documentation table of content at '
						    """`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the """
						    """others."""    )
				# Only add this once
				new_doc.append({"""local""": duplicate_key, """title""": titles[0]}    )

		# Add none duplicate-keys
		new_doc.extend([doc for doc in doc_list if """local""" not in counts or counts[doc["""local"""]] == 1]    )
		A__        =   sorted(__a    ,     key=lambda __a    : s["title"].lower()    )

		# "overview" gets special treatment and is always first
		if len(__a    ) > 1:
				raise ValueError("""{doc_list} has two 'overview' docs which is not allowed."""    )

		overview_doc.extend(__a    )

		# Sort
		return overview_doc







def    __lowerCamelCase					(							__a   :Any=False    )		->				List[str]:






		"""simple docstring"""

		with open(__a    ,     encoding="""utf-8"""    ) as f:
				A__        =   yaml.safe_load(f.read()    )

		# Get to the API doc
		A__        =   0
		while content[api_idx]["title"] != "API":
				api_idx += 1
		A__        =   content[api_idx]["""sections"""]

		# Then to the model doc
		A__        =   0
		while api_doc[scheduler_idx]["title"] != "Schedulers":
				scheduler_idx += 1

		A__        =   api_doc[scheduler_idx]["""sections"""]
		A__        =   clean_doc_toc(__a    )

		A__        =   False
		if new_scheduler_doc != scheduler_doc:
				A__        =   True
				if overwrite:
						A__        =   new_scheduler_doc

		if diff:
				if overwrite:
						A__        =   api_doc
						with open(__a    ,     """w"""    ,     encoding="""utf-8"""    ) as f:
								f.write(yaml.dump(__a    ,     allow_unicode=__a    )    )
				else:
						raise ValueError(
						    """The model doc part of the table of content is not properly sorted, run `make style` to fix this."""    )







def    __lowerCamelCase					(							__a   :Optional[int]=False    )		->				Dict:






		"""simple docstring"""

		with open(__a    ,     encoding="""utf-8"""    ) as f:
				A__        =   yaml.safe_load(f.read()    )

		# Get to the API doc
		A__        =   0
		while content[api_idx]["title"] != "API":
				api_idx += 1
		A__        =   content[api_idx]["""sections"""]

		# Then to the model doc
		A__        =   0
		while api_doc[pipeline_idx]["title"] != "Pipelines":
				pipeline_idx += 1

		A__        =   False
		A__        =   api_doc[pipeline_idx]["""sections"""]
		A__        =   []

		# sort sub pipeline docs
		for pipeline_doc in pipeline_docs:
				if "section" in pipeline_doc:
						A__        =   pipeline_doc["""section"""]
						A__        =   clean_doc_toc(__a    )
						if overwrite:
								A__        =   new_sub_pipeline_doc
				new_pipeline_docs.append(__a    )

		# sort overall pipeline doc
		A__        =   clean_doc_toc(__a    )

		if new_pipeline_docs != pipeline_docs:
				A__        =   True
				if overwrite:
						A__        =   new_pipeline_docs

		if diff:
				if overwrite:
						A__        =   api_doc
						with open(__a    ,     """w"""    ,     encoding="""utf-8"""    ) as f:
								f.write(yaml.dump(__a    ,     allow_unicode=__a    )    )
				else:
						raise ValueError(
						    """The model doc part of the table of content is not properly sorted, run `make style` to fix this."""    )


if __name__ == "__main__":
					A    :					Tuple =      argparse.ArgumentParser()
					parser.add_argument('''--fix_and_overwrite''', action='''store_true''', help='''Whether to fix inconsistencies.''')
					A    :					Optional[Any] =      parser.parse_args()

					check_scheduler_doc(args.fix_and_overwrite)
					check_pipeline_doc(args.fix_and_overwrite)



