


# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase=0	):


			'''simple docstring'''







			if name is None:
						__lowercase			       = None
			else:
						__lowercase			       = '''.''' * max(0							,      spaces - 2	) + '''# {:''' + str(50 - spaces	) + '''s}'''
						__lowercase			       = fmt.format(_UpperCamelCase	)

			# Print and recurse (if needed).
			if isinstance(_UpperCamelCase							,      _UpperCamelCase	):
						if msg is not None:
									print(_UpperCamelCase	)
						for k in val.keys():
									recursive_print(_UpperCamelCase							,      val[k]							,      spaces + 2	)
			elif isinstance(_UpperCamelCase							,      torch.Tensor	):
						print(_UpperCamelCase							,      ''':'''							,      val.size()	)
			else:
						print(_UpperCamelCase							,      ''':'''							,      _UpperCamelCase	)


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	):


			'''simple docstring'''







			__lowercase			       = param.size()
			if checkpoint_version == 1.0:
						# version 1.0 stores [num_heads * hidden_size * num_splits, :]
						__lowercase			       = (num_heads, hidden_size, num_splits) + input_shape[1:]
						__lowercase			       = param.view(*_UpperCamelCase	)
						__lowercase			       = param.transpose(0							,      2	)
						__lowercase			       = param.transpose(1							,      2	).contiguous()
			elif checkpoint_version >= 2.0:
						# other versions store [num_heads * num_splits * hidden_size, :]
						__lowercase			       = (num_heads, num_splits, hidden_size) + input_shape[1:]
						__lowercase			       = param.view(*_UpperCamelCase	)
						__lowercase			       = param.transpose(0							,      1	).contiguous()
			__lowercase			       = param.view(*_UpperCamelCase	)
			return param


def 		lowercase_	(  _UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	):


			'''simple docstring'''







			__lowercase			       = {}

			# old versions did not store training args
			__lowercase			       = input_state_dict.get('''args'''							,      _UpperCamelCase	)
			if ds_args is not None:
						# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
						# from pprint import pprint
						# pprint(vars(ds_args))

						__lowercase			       = ds_args.padded_vocab_size
						__lowercase			       = ds_args.max_position_embeddings
						__lowercase			       = ds_args.hidden_size
						__lowercase			       = ds_args.num_layers
						__lowercase			       = ds_args.num_attention_heads
						__lowercase			       = ds_args.ffn_hidden_size
						# pprint(config)

			# The number of heads.
			__lowercase			       = config.n_head
			# The hidden_size per head.
			__lowercase			       = config.n_embd // config.n_head
			# Megatron-LM checkpoint version
			if "checkpoint_version" in input_state_dict.keys():
						__lowercase			       = input_state_dict['''checkpoint_version''']
			else:
						__lowercase			       = 0.0

			# The model.
			__lowercase			       = input_state_dict['''model''']
			# The language model.
			__lowercase			       = model['''language_model''']
			# The embeddings.
			__lowercase			       = lm['''embedding''']

			# The word embeddings.
			__lowercase			       = embeddings['''word_embeddings''']['''weight''']
			# Truncate the embedding table to vocab_size rows.
			__lowercase			       = word_embeddings[: config.vocab_size, :]
			__lowercase			       = word_embeddings

			# The position embeddings.
			__lowercase			       = embeddings['''position_embeddings''']['''weight''']
			# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
			__lowercase			       = pos_embeddings.size(0	)
			if n_positions != config.n_positions:
						raise ValueError(
						    F'pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'	)
			# Store the position embeddings.
			__lowercase			       = pos_embeddings

			# The transformer.
			__lowercase			       = lm['''transformer'''] if '''transformer''' in lm.keys() else lm['''encoder''']

			# The regex to extract layer names.
			__lowercase			       = re.compile(r'''layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'''	)

			# The simple map of names for "automated" rules.
			__lowercase			       = {
			    '''attention.dense''': '''.attn.c_proj.''',
			    '''self_attention.dense''': '''.attn.c_proj.''',
			    '''mlp.dense_h_to_4h''': '''.mlp.c_fc.''',
			    '''mlp.dense_4h_to_h''': '''.mlp.c_proj.''',
			}

			# Extract the layers.
			for key, val in transformer.items():
						# Match the name.
						__lowercase			       = layer_re.match(_UpperCamelCase	)

						# Stop if that's not a layer
						if m is None:
									break

						# The index of the layer.
						__lowercase			       = int(m.group(1	)	)
						# The name of the operation.
						__lowercase			       = m.group(2	)
						# Is it a weight or a bias?
						__lowercase			       = m.group(3	)

						# The name of the layer.
						__lowercase			       = F'transformer.h.{layer_idx}'

						# For layernorm(s), simply store the layer norm.
						if op_name.endswith('''layernorm'''	):
									__lowercase			       = '''ln_1''' if op_name.startswith('''input'''	) else '''ln_2'''
									__lowercase			       = val

						# Transpose the QKV matrix.
						elif (
						    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
						) and weight_or_bias == "weight":
									# Insert a tensor of 1x1xDxD bias.
									__lowercase			       = torch.tril(torch.ones((n_positions, n_positions)							,      dtype=torch.floataa	)	).view(
									    1							,      1							,      _UpperCamelCase							,      _UpperCamelCase	)
									__lowercase			       = causal_mask

									# Insert a "dummy" tensor for masked_bias.
									__lowercase			       = torch.tensor(-1E4							,      dtype=torch.floataa	)
									__lowercase			       = masked_bias

									__lowercase			       = fix_query_key_value_ordering(_UpperCamelCase							,      _UpperCamelCase							,      3							,      _UpperCamelCase							,      _UpperCamelCase	)
									# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
									__lowercase			       = out_val.transpose(0							,      1	).contiguous()
									# Store.
									__lowercase			       = out_val

						# Transpose the bias.
						elif (
						    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
						) and weight_or_bias == "bias":
									__lowercase			       = fix_query_key_value_ordering(_UpperCamelCase							,      _UpperCamelCase							,      3							,      _UpperCamelCase							,      _UpperCamelCase	)
									# Store. No change of shape.
									__lowercase			       = out_val

						# Transpose the weights.
						elif weight_or_bias == "weight":
									__lowercase			       = megatron_to_transformers[op_name]
									__lowercase			       = val.transpose(0							,      1	)

						# Copy the bias.
						elif weight_or_bias == "bias":
									__lowercase			       = megatron_to_transformers[op_name]
									__lowercase			       = val

    # DEBUG.
			assert config.n_layer == layer_idx + 1

			# The final layernorm.
			__lowercase			       = transformer['''final_layernorm.weight''']
			__lowercase			       = transformer['''final_layernorm.bias''']

			# For LM head, transformers' wants the matrix to weight embeddings.
			__lowercase			       = word_embeddings

			# It should be done!
			return output_state_dict


def 		lowercase_	(  ):


			'''simple docstring'''







			__lowercase			       = argparse.ArgumentParser()
			parser.add_argument('''--print-checkpoint-structure'''							,      action='''store_true'''	)
			parser.add_argument(
			    '''path_to_checkpoint'''							,      type=_UpperCamelCase							,      help='''Path to the checkpoint file (.zip archive or direct .pt file)'''							,      )
			parser.add_argument(
			    '''--config_file'''							,      default=''''''							,      type=_UpperCamelCase							,      help='''An optional config json file describing the pre-trained model.'''							,      )
			__lowercase			       = parser.parse_args()

			# Extract the basename.
			__lowercase			       = os.path.dirname(args.path_to_checkpoint	)

			# Load the model.
			# the .zip is very optional, let's keep it for backward compatibility
			print(F'Extracting PyTorch state dictionary from {args.path_to_checkpoint}'	)
			if args.path_to_checkpoint.endswith('''.zip'''	):
						with zipfile.ZipFile(args.path_to_checkpoint							,      '''r'''	) as checkpoint:
									with checkpoint.open('''release/mp_rank_00/model_optim_rng.pt'''	) as pytorch_dict:
												__lowercase			       = torch.load(_UpperCamelCase							,      map_location='''cpu'''	)
			else:
						__lowercase			       = torch.load(args.path_to_checkpoint							,      map_location='''cpu'''	)

			__lowercase			       = input_state_dict.get('''args'''							,      _UpperCamelCase	)

			# Read the config, or default to the model released by NVIDIA.
			if args.config_file == "":
						if ds_args is not None:
									if ds_args.bias_gelu_fusion:
												__lowercase			       = '''gelu_fast'''
									elif ds_args.openai_gelu:
												__lowercase			       = '''gelu_new'''
									else:
												__lowercase			       = '''gelu'''
						else:
									# in the very early days this used to be "gelu_new"
									__lowercase			       = '''gelu_new'''

						# Spell out all parameters in case the defaults change.
						__lowercase			       = GPTaConfig(
						    vocab_size=5_02_57							,      n_positions=10_24							,      n_embd=10_24							,      n_layer=24							,      n_head=16							,      n_inner=40_96							,      activation_function=_UpperCamelCase							,      resid_pdrop=0.1							,      embd_pdrop=0.1							,      attn_pdrop=0.1							,      layer_norm_epsilon=1E-5							,      initializer_range=0.02							,      summary_type='''cls_index'''							,      summary_use_proj=_UpperCamelCase							,      summary_activation=_UpperCamelCase							,      summary_proj_to_labels=_UpperCamelCase							,      summary_first_dropout=0.1							,      scale_attn_weights=_UpperCamelCase							,      use_cache=_UpperCamelCase							,      bos_token_id=5_02_56							,      eos_token_id=5_02_56							,      )
			else:
						__lowercase			       = GPTaConfig.from_json_file(args.config_file	)

			__lowercase			       = ['''GPT2LMHeadModel''']

			# Convert.
			print('''Converting'''	)
			__lowercase			       = convert_megatron_checkpoint(_UpperCamelCase							,      _UpperCamelCase							,      _UpperCamelCase	)

			# Print the structure of converted state dict.
			if args.print_checkpoint_structure:
						recursive_print(_UpperCamelCase							,      _UpperCamelCase	)

			# Add tokenizer class info to config
			# see https://github.com/huggingface/transformers/issues/13906)
			if ds_args is not None:
						__lowercase			       = ds_args.tokenizer_type
						if tokenizer_type == "GPT2BPETokenizer":
									__lowercase			       = '''gpt2'''
						elif tokenizer_type == "PretrainedFromHF":
									__lowercase			       = ds_args.tokenizer_name_or_path
						else:
									raise ValueError(F'Unrecognized tokenizer_type {tokenizer_type}'	)
			else:
						__lowercase			       = '''gpt2'''

			__lowercase			       = AutoTokenizer.from_pretrained(_UpperCamelCase	)
			__lowercase			       = type(_UpperCamelCase	).__name__
			__lowercase			       = tokenizer_class

			# Store the config to file.
			print('''Saving config'''	)
			config.save_pretrained(_UpperCamelCase	)

			# Save tokenizer based on args
			print(F'Adding {tokenizer_class} tokenizer files'	)
			tokenizer.save_pretrained(_UpperCamelCase	)

			# Store the state_dict to file.
			__lowercase			       = os.path.join(_UpperCamelCase							,      '''pytorch_model.bin'''	)
			print(F'Saving checkpoint to "{output_checkpoint_file}"'	)
			torch.save(_UpperCamelCase							,      _UpperCamelCase	)


####################################################################################################

if __name__ == "__main__":
	main()

####################################################################################################






# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

a			:			Dict												=					'''2.13.1'''

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse('''3.7'''):
	raise ImportWarning(
	    '''To use `datasets`, Python>=3.7 is required, and the current version of Python doesn\'t match this condition.'''
	)

if version.parse(pyarrow.__version__).major < 8:
	raise ImportWarning(
	    '''To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn\'t match this condition.\n'''
	    '''If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`.'''
	)

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

a			:			Optional[int]												=					concatenate_datasets
a			:			List[Any]												=					DownloadConfig
a			:			List[Any]												=					DownloadManager
a			:			str												=					DownloadMode
a			:			int												=					DownloadConfig
a			:			List[str]												=					DownloadMode
a			:			Optional[int]												=					DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager


