





import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

lowerCAmelCase_    =				[
    '''word_embeddings_layernorm.weight''',
    '''word_embeddings_layernorm.bias''',
    '''input_layernorm.weight''',
    '''input_layernorm.bias''',
    '''post_attention_layernorm.weight''',
    '''post_attention_layernorm.bias''',
    '''self_attention.dense.bias''',
    '''mlp.dense_4h_to_h.bias''',
    '''ln_f.weight''',
    '''ln_f.bias''',
]

lowerCAmelCase_    =				[
    '''mlp.dense_4h_to_h.weight''',
    '''self_attention.dense.weight''',
]






def      lowerCamelCase_						(				_UpperCamelCase ,				_UpperCamelCase )		->						List[Any]:






				"""simple docstring"""


				snake_case_						:    Tuple				=  {
				    '''word_embeddings.weight''': '''word_embeddings.weight''',
				    '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
				    '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
				    '''weight''': '''ln_f.weight''',
				    '''bias''': '''ln_f.bias''',
				}

				if key in layer_rename_map:
								return layer_rename_map[key]

				# Handle transformer blocks
				snake_case_						:    Dict				=  int(re.match(R'''.*layer_(\d*).*''' ,				_UpperCamelCase )[1] )
				layer_number -= 3
				return f'''h.{layer_number}.''' + key






def      lowerCamelCase_						(				_UpperCamelCase )		->						Optional[Any]:






				"""simple docstring"""


				if dtype == torch.bool:
								return 1 / 8
				snake_case_						:    List[str]				=  re.search(R'''[^\d](\d+)$''' ,				str(_UpperCamelCase ) )
				if bit_search is None:
								raise ValueError(f'''`dtype` is not a valid dtype: {dtype}.''' )
				snake_case_						:    Dict				=  int(bit_search.groups()[0] )
				return bit_size // 8






def      lowerCamelCase_						(				_UpperCamelCase ,				_UpperCamelCase ,				_UpperCamelCase ,				_UpperCamelCase ,				_UpperCamelCase )		->						Union[str, Any]:






				"""simple docstring"""


				if bloom_config_file == "":
								snake_case_						:    Tuple				=  BloomConfig()
				else:
								snake_case_						:    List[Any]				=  BloomConfig.from_json_file(_UpperCamelCase )

				if shard_model:
								snake_case_						:    Optional[int]				=  os.listdir(_UpperCamelCase )
								snake_case_						:    str				=  sorted(filter(lambda _UpperCamelCase : s.startswith('''layer''' ) and "model_00" in s ,				_UpperCamelCase ) )

								snake_case_						:    int				=  {'''weight_map''': {}, '''metadata''': {}}
								snake_case_						:    Any				=  0

								snake_case_						:    Any				=  None

								snake_case_						:    Optional[Any]				=  BloomConfig()

								for j, file in enumerate(_UpperCamelCase ):
												print('''Processing file: {}'''.format(_UpperCamelCase ) )
												snake_case_						:    Optional[Any]				=  None

												for i in range(_UpperCamelCase ):
																# load all TP files
																snake_case_						:    Dict				=  file.replace('''model_00''' ,				f'''model_0{i}''' )
																snake_case_						:    List[str]				=  torch.load(os.path.join(_UpperCamelCase ,				_UpperCamelCase ) ,				map_location='''cpu''' )

																# Rename keys in the transformers names
																snake_case_						:    Tuple				=  list(temp.keys() )
																for key in keys:
																				snake_case_						:    Tuple				=  temp.pop(_UpperCamelCase )

																if tensors is None:
																				snake_case_						:    Tuple				=  temp
																else:
																				for key in tensors.keys():
																								if any(key.endswith(_UpperCamelCase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
																												# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																												tensors[key] += temp[key]
																								else:
																												# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																												snake_case_						:    Dict				=  1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN ) else 0
																												# We concatenate these weights accross TP ranks
																												snake_case_						:    Any				=  torch.cat([tensors[key], temp[key]] ,				dim=_UpperCamelCase )

            # Divide by the number of TP the weights we want to average
												for key in tensors.keys():
																if any(key.endswith(_UpperCamelCase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
																				snake_case_						:    Optional[int]				=  tensors[key] / pretraining_tp
												torch.save(
												    _UpperCamelCase ,				os.path.join(
												        _UpperCamelCase ,				'''pytorch_model_{}-of-{}.bin'''.format(str(j + 1 ).zfill(5 ) ,				str(len(_UpperCamelCase ) ).zfill(5 ) ) ,				) ,				)

												for key in tensors.keys():
																snake_case_						:    Dict				=  tensors[key]
																total_size += value.numel() * get_dtype_size(value.dtype )
																if key not in index_dict["weight_map"]:
																				snake_case_						:    int				=  '''pytorch_model_{}-of-{}.bin'''.format(
																				    str(j + 1 ).zfill(5 ) ,				str(len(_UpperCamelCase ) ).zfill(5 ) )

								snake_case_						:    Union[str, Any]				=  BloomConfig()
								snake_case_						:    List[str]				=  pytorch_dump_folder_path + '''/''' + CONFIG_NAME
								snake_case_						:    Optional[Any]				=  total_size
								with open(_UpperCamelCase ,				'''w''' ,				encoding='''utf-8''' ) as f:
												f.write(config.to_json_string() )
								with open(os.path.join(_UpperCamelCase ,				WEIGHTS_NAME + '''.index.json''' ) ,				'''w''' ,				encoding='''utf-8''' ) as f:
												snake_case_						:    Optional[int]				=  json.dumps(_UpperCamelCase ,				indent=2 ,				sort_keys=_UpperCamelCase ) + '''\n'''
												f.write(_UpperCamelCase )
				else:
								snake_case_						:    Any				=  BloomModel(_UpperCamelCase )

								snake_case_						:    Dict				=  os.listdir(_UpperCamelCase )
								snake_case_						:    List[Any]				=  sorted(filter(lambda _UpperCamelCase : s.startswith('''layer''' ) and "model_00" in s ,				_UpperCamelCase ) )

								snake_case_						:    Any				=  None
								for i, file in enumerate(_UpperCamelCase ):
												snake_case_						:    Union[str, Any]				=  None
												for i in range(_UpperCamelCase ):
																# load all TP files
																snake_case_						:    Tuple				=  file.replace('''model_00''' ,				f'''model_0{i}''' )
																snake_case_						:    int				=  torch.load(os.path.join(_UpperCamelCase ,				_UpperCamelCase ) ,				map_location='''cpu''' )

																# Rename keys in the transformers names
																snake_case_						:    Union[str, Any]				=  list(temp.keys() )
																for key in keys:
																				snake_case_						:    Optional[int]				=  temp.pop(_UpperCamelCase )

																if tensors is None:
																				snake_case_						:    str				=  temp
																else:
																				for key in tensors.keys():
																								# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																								if any(key.endswith(_UpperCamelCase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
																												tensors[key] += temp[key]
																								else:
																												# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																												snake_case_						:    Optional[Any]				=  1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN ) else 0
																												# We concatenate these weights accross TP ranks
																												snake_case_						:    Any				=  torch.cat([tensors[key], temp[key]] ,				dim=_UpperCamelCase )

            # Divide by the number of TP the weights we want to average
												for key in tensors.keys():
																if any(key.endswith(_UpperCamelCase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
																				snake_case_						:    int				=  tensors[key] / pretraining_tp

												snake_case_						:    Tuple				=  model.load_state_dict(_UpperCamelCase ,				strict=_UpperCamelCase )
												assert not other_keys.unexpected_keys, f'''The keys {other_keys.unexpected_keys} are unexpected'''
												if missing_keys is None:
																snake_case_						:    int				=  set(other_keys.missing_keys )
												else:
																snake_case_						:    Dict				=  missing_keys.intersection(set(other_keys.missing_keys ) )

								assert not missing_keys, f'''The keys {missing_keys} are missing'''

								# Save pytorch-model
								os.makedirs(_UpperCamelCase ,				exist_ok=_UpperCamelCase )
								snake_case_						:    Optional[Any]				=  pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
								snake_case_						:    Any				=  pytorch_dump_folder_path + '''/''' + CONFIG_NAME
								print(f'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}''' )
								if config.torch_dtype is not None:
												snake_case_						:    List[str]				=  model.to(config.torch_dtype )
								torch.save(model.state_dict() ,				_UpperCamelCase )
								print(f'''Save configuration file to {pytorch_config_dump_path}''' )
								with open(_UpperCamelCase ,				'''w''' ,				encoding='''utf-8''' ) as f:
												f.write(config.to_json_string() )


if __name__ == "__main__":
	lowerCAmelCase_    =				argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '''--bloom_checkpoint_path''',
	    default=None,
	    type=str,
	    required=True,
	    help='''Path to the Megatron-LM checkpoint path.''',
	)
	parser.add_argument(
	    '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
	)
	parser.add_argument(
	    '''--bloom_config_file''',
	    default='''''',
	    type=str,
	    help=(
	        '''An optional config json file corresponding to the pre-trained model. \n'''
	        '''This specifies the model architecture.'''
	    ),
	)
	parser.add_argument(
	    '''--shard_model''',
	    action='''store_true''',
	    help='''An optional setting to shard the output model \nThis enables sharding the converted checkpoint''',
	)
	parser.add_argument(
	    '''--pretraining_tp''',
	    default=4,
	    type=int,
	    help='''Pretraining TP rank that has been used when training the model in Megatron-LM \n''',
	)
	lowerCAmelCase_    =				parser.parse_args()
	convert_bloom_checkpoint_to_pytorch(
	    args.bloom_checkpoint_path,
	    args.bloom_config_file,
	    args.pytorch_dump_folder_path,
	    args.shard_model,
	    args.pretraining_tp,
	)







import argparse

from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection

from diffusers import UnCLIPImageVariationPipeline, UnCLIPPipeline


if __name__ == "__main__":
	lowerCAmelCase_    =				argparse.ArgumentParser()

	parser.add_argument('''--dump_path''', default=None, type=str, required=True, help='''Path to the output model.''')

	parser.add_argument(
	    '''--txt2img_unclip''',
	    default='''kakaobrain/karlo-v1-alpha''',
	    type=str,
	    required=False,
	    help='''The pretrained txt2img unclip.''',
	)

	lowerCAmelCase_    =				parser.parse_args()

	lowerCAmelCase_    =				UnCLIPPipeline.from_pretrained(args.txtaimg_unclip)

	lowerCAmelCase_    =				CLIPImageProcessor()
	lowerCAmelCase_    =				CLIPVisionModelWithProjection.from_pretrained('''openai/clip-vit-large-patch14''')

	lowerCAmelCase_    =				UnCLIPImageVariationPipeline(
	    decoder=txtaimg.decoder,
	    text_encoder=txtaimg.text_encoder,
	    tokenizer=txtaimg.tokenizer,
	    text_proj=txtaimg.text_proj,
	    feature_extractor=feature_extractor,
	    image_encoder=image_encoder,
	    super_res_first=txtaimg.super_res_first,
	    super_res_last=txtaimg.super_res_last,
	    decoder_scheduler=txtaimg.decoder_scheduler,
	    super_res_scheduler=txtaimg.super_res_scheduler,
	)

	imgaimg.save_pretrained(args.dump_path)
