





"""simple docstring"""
import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

_snake_case     :		List[Any]       						= [
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

_snake_case     :		Optional[Any]       						= [
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]






def 					A__				(			UpperCamelCase					,  UpperCamelCase       ):
							A		=  {
							    "word_embeddings.weight": "word_embeddings.weight",
							    "word_embeddings.norm.weight": "word_embeddings_layernorm.weight",
							    "word_embeddings.norm.bias": "word_embeddings_layernorm.bias",
							    "weight": "ln_f.weight",
							    "bias": "ln_f.bias",
							}

							if key in layer_rename_map:
														return layer_rename_map[key]

							# Handle transformer blocks
							A		=  int(re.match(r".*layer_(\d*).*"					,  UpperCamelCase       )[1]       )
							layer_number -= 3
							return F"h.{layer_number}." + key






def 					A__				(			UpperCamelCase       ):
							if dtype == torch.bool:
														return 1 / 8
							A		=  re.search(r"[^\d](\d+)$"					,  str(UpperCamelCase       )       )
							if bit_search is None:
														raise ValueError(F"`dtype` is not a valid dtype: {dtype}."       )
							A		=  int(bit_search.groups()[0]       )
							return bit_size // 8






def 					A__				(			UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase       ):
							# Construct model
							if bloom_config_file == "":
														A		=  BloomConfig()
							else:
														A		=  BloomConfig.from_json_file(UpperCamelCase       )

							if shard_model:
														A		=  os.listdir(UpperCamelCase       )
														A		=  sorted(filter(lambda UpperCamelCase       : s.startswith("layer"       ) and "model_00" in s					,  UpperCamelCase       )       )

														A		=  {"weight_map": {}, "metadata": {}}
														A		=  0

														A		=  None

														A		=  BloomConfig()

														for j, file in enumerate(UpperCamelCase       ):
																					print("Processing file: {}".format(UpperCamelCase       )       )
																					A		=  None

																					for i in range(UpperCamelCase       ):
																												# load all TP files
																												A		=  file.replace("model_00"					,  F"model_0{i}"       )
																												A		=  torch.load(os.path.join(UpperCamelCase					,  UpperCamelCase       )					,  map_location="cpu"       )

																												# Rename keys in the transformers names
																												A		=  list(temp.keys()       )
																												for key in keys:
																																			A		=  temp.pop(UpperCamelCase       )

																												if tensors is None:
																																			A		=  temp
																												else:
																																			for key in tensors.keys():
																																										if any(key.endswith(UpperCamelCase       ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH       ):
																																																	# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																																	tensors[key] += temp[key]
																																										else:
																																																	# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																																	A		=  1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN       ) else 0
																																																	# We concatenate these weights accross TP ranks
																																																	A		=  torch.cat([tensors[key], temp[key]]					,  dim=UpperCamelCase       )

            # Divide by the number of TP the weights we want to average
																					for key in tensors.keys():
																												if any(key.endswith(UpperCamelCase       ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH       ):
																																			A		=  tensors[key] / pretraining_tp
																					torch.save(
																					    UpperCamelCase					,  os.path.join(
																					        UpperCamelCase					,  "pytorch_model_{}-of-{}.bin".format(str(j + 1       ).zfill(5       )					,  str(len(UpperCamelCase       )       ).zfill(5       )       )					,  )					,  )

																					for key in tensors.keys():
																												A		=  tensors[key]
																												total_size += value.numel() * get_dtype_size(value.dtype       )
																												if key not in index_dict["weight_map"]:
																																			A		=  "pytorch_model_{}-of-{}.bin".format(
																																			    str(j + 1       ).zfill(5       )					,  str(len(UpperCamelCase       )       ).zfill(5       )       )

														A		=  BloomConfig()
														A		=  pytorch_dump_folder_path + "/" + CONFIG_NAME
														A		=  total_size
														with open(UpperCamelCase					,  "w"					,  encoding="utf-8"       ) as f:
																					f.write(config.to_json_string()       )
														with open(os.path.join(UpperCamelCase					,  WEIGHTS_NAME + ".index.json"       )					,  "w"					,  encoding="utf-8"       ) as f:
																					A		=  json.dumps(UpperCamelCase					,  indent=2					,  sort_keys=UpperCamelCase       ) + "\n"
																					f.write(UpperCamelCase       )
							else:
														A		=  BloomModel(UpperCamelCase       )

														A		=  os.listdir(UpperCamelCase       )
														A		=  sorted(filter(lambda UpperCamelCase       : s.startswith("layer"       ) and "model_00" in s					,  UpperCamelCase       )       )

														A		=  None
														for i, file in enumerate(UpperCamelCase       ):
																					A		=  None
																					for i in range(UpperCamelCase       ):
																												# load all TP files
																												A		=  file.replace("model_00"					,  F"model_0{i}"       )
																												A		=  torch.load(os.path.join(UpperCamelCase					,  UpperCamelCase       )					,  map_location="cpu"       )

																												# Rename keys in the transformers names
																												A		=  list(temp.keys()       )
																												for key in keys:
																																			A		=  temp.pop(UpperCamelCase       )

																												if tensors is None:
																																			A		=  temp
																												else:
																																			for key in tensors.keys():
																																										# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																										if any(key.endswith(UpperCamelCase       ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH       ):
																																																	tensors[key] += temp[key]
																																										else:
																																																	# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																																	A		=  1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN       ) else 0
																																																	# We concatenate these weights accross TP ranks
																																																	A		=  torch.cat([tensors[key], temp[key]]					,  dim=UpperCamelCase       )

            # Divide by the number of TP the weights we want to average
																					for key in tensors.keys():
																												if any(key.endswith(UpperCamelCase       ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH       ):
																																			A		=  tensors[key] / pretraining_tp

																					A		=  model.load_state_dict(UpperCamelCase					,  strict=UpperCamelCase       )
																					assert not other_keys.unexpected_keys, F"The keys {other_keys.unexpected_keys} are unexpected"
																					if missing_keys is None:
																												A		=  set(other_keys.missing_keys       )
																					else:
																												A		=  missing_keys.intersection(set(other_keys.missing_keys       )       )

														assert not missing_keys, F"The keys {missing_keys} are missing"

														# Save pytorch-model
														os.makedirs(UpperCamelCase					,  exist_ok=UpperCamelCase       )
														A		=  pytorch_dump_folder_path + "/" + WEIGHTS_NAME
														A		=  pytorch_dump_folder_path + "/" + CONFIG_NAME
														print(F"Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}"       )
														if config.torch_dtype is not None:
																					A		=  model.to(config.torch_dtype       )
														torch.save(model.state_dict()					,  UpperCamelCase       )
														print(F"Save configuration file to {pytorch_config_dump_path}"       )
														with open(UpperCamelCase					,  "w"					,  encoding="utf-8"       ) as f:
																					f.write(config.to_json_string()       )


if __name__ == "__main__":
		_snake_case     :		Optional[int]       						= argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '--bloom_checkpoint_path',
		    default=None,
		    type=str,
		    required=True,
		    help='Path to the Megatron-LM checkpoint path.',
		)
		parser.add_argument(
		    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
		)
		parser.add_argument(
		    '--bloom_config_file',
		    default='',
		    type=str,
		    help=(
		        'An optional config json file corresponding to the pre-trained model. \n'
		        'This specifies the model architecture.'
		    ),
		)
		parser.add_argument(
		    '--shard_model',
		    action='store_true',
		    help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
		)
		parser.add_argument(
		    '--pretraining_tp',
		    default=4,
		    type=int,
		    help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
		)
		_snake_case     :		str       						= parser.parse_args()
		convert_bloom_checkpoint_to_pytorch(
		    args.bloom_checkpoint_path,
		    args.bloom_config_file,
		    args.pytorch_dump_folder_path,
		    args.shard_model,
		    args.pretraining_tp,
		)









"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case     :		Optional[int]       						= logging.get_logger(__name__)

_snake_case     :		Optional[int]       						= {
    'google/vivit-b-16x2-kinetics400': (
        'https://huggingface.co/google/vivit-b-16x2-kinetics400/resolve/main/config.json'
    ),
    # See all Vivit models at https://huggingface.co/models?filter=vivit
}






class 					_UpperCAmelCase   (	lowercase_ ):
					UpperCamelCase      =							'''vivit'''


					def __init__(					self			:Optional[Any]     ,       __UpperCamelCase			:Dict=2_24     ,       __UpperCamelCase			:int=32     ,       __UpperCamelCase			:Union[str, Any]=[2, 16, 16]     ,       __UpperCamelCase			:Optional[Any]=3     ,       __UpperCamelCase			:Optional[Any]=7_68     ,       __UpperCamelCase			:Any=12     ,       __UpperCamelCase			:List[str]=12     ,       __UpperCamelCase			:List[str]=30_72     ,       __UpperCamelCase			:Any="gelu_fast"     ,       __UpperCamelCase			:List[Any]=0.0     ,       __UpperCamelCase			:str=0.0     ,       __UpperCamelCase			:Dict=0.02     ,       __UpperCamelCase			:Optional[Any]=1e-06     ,       __UpperCamelCase			:Dict=True     ,       **__UpperCamelCase			:Tuple     ,       ):
												A		=  hidden_size
												A		=  num_hidden_layers
												A		=  num_attention_heads
												A		=  intermediate_size
												A		=  hidden_act
												A		=  hidden_dropout_prob
												A		=  attention_probs_dropout_prob
												A		=  initializer_range
												A		=  layer_norm_eps

												A		=  image_size
												A		=  num_frames
												A		=  tubelet_size
												A		=  num_channels
												A		=  qkv_bias

												super().__init__(**__UpperCamelCase		)


