




'''simple docstring'''

import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

a    :							str										=    [
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

a    :							List[str]										=    [
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]


def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase      )						->   Optional[Any]:

					'''simple docstring'''

					snake_case_		     =							{
					    '''word_embeddings.weight''': '''word_embeddings.weight''',
					    '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
					    '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
					    '''weight''': '''ln_f.weight''',
					    '''bias''': '''ln_f.bias''',
					}

					if key in layer_rename_map:
										return layer_rename_map[key]

					# Handle transformer blocks
					snake_case_		     =							int(re.match(r'''.*layer_(\d*).*''', __UpperCAmelCase      )[1]      )
					layer_number -= 3
					return F"h.{layer_number}." + key


def 						__magic_name__       (      __UpperCAmelCase      )						->   Dict:

					'''simple docstring'''

					if dtype == torch.bool:
										return 1 / 8
					snake_case_		     =							re.search(r'''[^\d](\d+)$''', str(__UpperCAmelCase      )      )
					if bit_search is None:
										raise ValueError(F"`dtype` is not a valid dtype: {dtype}."      )
					snake_case_		     =							int(bit_search.groups()[0]      )
					return bit_size // 8


def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase      )						->   Union[str, Any]:

					'''simple docstring'''

					if bloom_config_file == "":
										snake_case_		     =							BloomConfig()
					else:
										snake_case_		     =							BloomConfig.from_json_file(__UpperCAmelCase      )

					if shard_model:
										snake_case_		     =							os.listdir(__UpperCAmelCase      )
										snake_case_		     =							sorted(filter(lambda __UpperCAmelCase      : s.startswith('''layer'''      ) and "model_00" in s, __UpperCAmelCase      )      )

										snake_case_		     =							{'''weight_map''': {}, '''metadata''': {}}
										snake_case_		     =							0

										snake_case_		     =							None

										snake_case_		     =							BloomConfig()

										for j, file in enumerate(__UpperCAmelCase      ):
															print('''Processing file: {}'''.format(__UpperCAmelCase      )      )
															snake_case_		     =							None

															for i in range(__UpperCAmelCase      ):
																				# load all TP files
																				snake_case_		     =							file.replace('''model_00''', F"model_0{i}"      )
																				snake_case_		     =							torch.load(os.path.join(__UpperCAmelCase, __UpperCAmelCase      ), map_location='''cpu'''      )

																				# Rename keys in the transformers names
																				snake_case_		     =							list(temp.keys()      )
																				for key in keys:
																									snake_case_		     =							temp.pop(__UpperCAmelCase      )

																				if tensors is None:
																									snake_case_		     =							temp
																				else:
																									for key in tensors.keys():
																														if any(key.endswith(__UpperCAmelCase      ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH      ):
																																			# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																			tensors[key] += temp[key]
																														else:
																																			# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																			snake_case_		     =							1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN      ) else 0
																																			# We concatenate these weights accross TP ranks
																																			snake_case_		     =							torch.cat([tensors[key], temp[key]], dim=__UpperCAmelCase      )

            # Divide by the number of TP the weights we want to average
															for key in tensors.keys():
																				if any(key.endswith(__UpperCAmelCase      ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH      ):
																									snake_case_		     =							tensors[key] / pretraining_tp
															torch.save(
															    __UpperCAmelCase, os.path.join(
															        __UpperCAmelCase, '''pytorch_model_{}-of-{}.bin'''.format(str(j + 1      ).zfill(5      ), str(len(__UpperCAmelCase      )      ).zfill(5      )      ), ), )

															for key in tensors.keys():
																				snake_case_		     =							tensors[key]
																				total_size += value.numel() * get_dtype_size(value.dtype      )
																				if key not in index_dict["weight_map"]:
																									snake_case_		     =							'''pytorch_model_{}-of-{}.bin'''.format(
																									    str(j + 1      ).zfill(5      ), str(len(__UpperCAmelCase      )      ).zfill(5      )      )

										snake_case_		     =							BloomConfig()
										snake_case_		     =							pytorch_dump_folder_path + '''/''' + CONFIG_NAME
										snake_case_		     =							total_size
										with open(__UpperCAmelCase, '''w''', encoding='''utf-8'''      ) as f:
															f.write(config.to_json_string()      )
										with open(os.path.join(__UpperCAmelCase, WEIGHTS_NAME + '''.index.json'''      ), '''w''', encoding='''utf-8'''      ) as f:
															snake_case_		     =							json.dumps(__UpperCAmelCase, indent=2, sort_keys=__UpperCAmelCase      ) + '''\n'''
															f.write(__UpperCAmelCase      )
					else:
										snake_case_		     =							BloomModel(__UpperCAmelCase      )

										snake_case_		     =							os.listdir(__UpperCAmelCase      )
										snake_case_		     =							sorted(filter(lambda __UpperCAmelCase      : s.startswith('''layer'''      ) and "model_00" in s, __UpperCAmelCase      )      )

										snake_case_		     =							None
										for i, file in enumerate(__UpperCAmelCase      ):
															snake_case_		     =							None
															for i in range(__UpperCAmelCase      ):
																				# load all TP files
																				snake_case_		     =							file.replace('''model_00''', F"model_0{i}"      )
																				snake_case_		     =							torch.load(os.path.join(__UpperCAmelCase, __UpperCAmelCase      ), map_location='''cpu'''      )

																				# Rename keys in the transformers names
																				snake_case_		     =							list(temp.keys()      )
																				for key in keys:
																									snake_case_		     =							temp.pop(__UpperCAmelCase      )

																				if tensors is None:
																									snake_case_		     =							temp
																				else:
																									for key in tensors.keys():
																														# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																														if any(key.endswith(__UpperCAmelCase      ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH      ):
																																			tensors[key] += temp[key]
																														else:
																																			# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																			snake_case_		     =							1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN      ) else 0
																																			# We concatenate these weights accross TP ranks
																																			snake_case_		     =							torch.cat([tensors[key], temp[key]], dim=__UpperCAmelCase      )

            # Divide by the number of TP the weights we want to average
															for key in tensors.keys():
																				if any(key.endswith(__UpperCAmelCase      ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH      ):
																									snake_case_		     =							tensors[key] / pretraining_tp

															snake_case_		     =							model.load_state_dict(__UpperCAmelCase, strict=__UpperCAmelCase      )
															assert not other_keys.unexpected_keys, F"The keys {other_keys.unexpected_keys} are unexpected"
															if missing_keys is None:
																				snake_case_		     =							set(other_keys.missing_keys      )
															else:
																				snake_case_		     =							missing_keys.intersection(set(other_keys.missing_keys      )      )

										assert not missing_keys, F"The keys {missing_keys} are missing"

										# Save pytorch-model
										os.makedirs(__UpperCAmelCase, exist_ok=__UpperCAmelCase      )
										snake_case_		     =							pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
										snake_case_		     =							pytorch_dump_folder_path + '''/''' + CONFIG_NAME
										print(F"Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}"      )
										if config.torch_dtype is not None:
															snake_case_		     =							model.to(config.torch_dtype      )
										torch.save(model.state_dict(), __UpperCAmelCase      )
										print(F"Save configuration file to {pytorch_config_dump_path}"      )
										with open(__UpperCAmelCase, '''w''', encoding='''utf-8'''      ) as f:
															f.write(config.to_json_string()      )


if __name__ == "__main__":
							a    :							Optional[Any]										=    argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '--bloom_checkpoint_path',
							    default=None,
							    type=str,
							    required=True,
							    help='Path to the Megatron-LM checkpoint path.',
							)
							parser.add_argument(
							    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
							)
							parser.add_argument(
							    '--bloom_config_file',
							    default='',
							    type=str,
							    help=(
							        'An optional config json file corresponding to the pre-trained model. \n'
							        'This specifies the model architecture.'
							    ),
							)
							parser.add_argument(
							    '--shard_model',
							    action='store_true',
							    help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
							)
							parser.add_argument(
							    '--pretraining_tp',
							    default=4,
							    type=int,
							    help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
							)
							a    :							Optional[Any]										=    parser.parse_args()
							convert_bloom_checkpoint_to_pytorch(
							    args.bloom_checkpoint_path,
							    args.bloom_config_file,
							    args.pytorch_dump_folder_path,
							    args.shard_model,
							    args.pretraining_tp,
							)







'''simple docstring'''

import argparse
from collections import defaultdict

import yaml


a    :							List[Any]										=    'docs/source/en/_toctree.yml'


def 						__magic_name__       (      __UpperCAmelCase      )						->   str:

					'''simple docstring'''

					snake_case_		     =							defaultdict(__UpperCAmelCase      )
					for doc in model_doc:
										counts[doc["local"]] += 1
					snake_case_		     =							[key for key, value in counts.items() if value > 1]

					snake_case_		     =							[]
					for duplicate_key in duplicates:
										snake_case_		     =							list({doc['''title'''] for doc in model_doc if doc['''local'''] == duplicate_key}      )
										if len(__UpperCAmelCase      ) > 1:
															raise ValueError(
															    F"{duplicate_key} is present several times in the documentation table of content at "
															    '''`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the '''
															    '''others.'''      )
										# Only add this once
										new_doc.append({'''local''': duplicate_key, '''title''': titles[0]}      )

					# Add none duplicate-keys
					new_doc.extend([doc for doc in model_doc if counts[doc['''local''']] == 1]      )

					# Sort
					return sorted(__UpperCAmelCase, key=lambda __UpperCAmelCase      : s["title"].lower()      )


def 						__magic_name__       (      __UpperCAmelCase=False      )						->   List[Any]:

					'''simple docstring'''

					with open(__UpperCAmelCase, encoding='''utf-8'''      ) as f:
										snake_case_		     =							yaml.safe_load(f.read()      )

					# Get to the API doc
					snake_case_		     =							0
					while content[api_idx]["title"] != "API":
										api_idx += 1
					snake_case_		     =							content[api_idx]['''sections''']

					# Then to the model doc
					snake_case_		     =							0
					while api_doc[model_idx]["title"] != "Models":
										model_idx += 1

					snake_case_		     =							api_doc[model_idx]['''sections''']

					snake_case_		     =							[(idx, section) for idx, section in enumerate(__UpperCAmelCase      ) if '''sections''' in section]
					snake_case_		     =							False
					for idx, modality_doc in modalities_docs:
										snake_case_		     =							modality_doc['''sections''']
										snake_case_		     =							clean_model_doc_toc(__UpperCAmelCase      )

										if old_modality_doc != new_modality_doc:
															snake_case_		     =							True
															if overwrite:
																				snake_case_		     =							new_modality_doc

					if diff:
										if overwrite:
															snake_case_		     =							model_doc
															snake_case_		     =							api_doc
															with open(__UpperCAmelCase, '''w''', encoding='''utf-8'''      ) as f:
																				f.write(yaml.dump(__UpperCAmelCase, allow_unicode=__UpperCAmelCase      )      )
										else:
															raise ValueError(
															    '''The model doc part of the table of content is not properly sorted, run `make style` to fix this.'''      )


if __name__ == "__main__":
							a    :							Dict										=    argparse.ArgumentParser()
							parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
							a    :							Union[str, Any]										=    parser.parse_args()

							check_model_doc(args.fix_and_overwrite)

