



"""simple docstring"""

from __future__ import annotations

lowercase__    :  Any							    =	10




def __lowercase			(							_a    ):
    snake_case_ :     str				=	1
    snake_case_ :     str				=	max(_a    )
    while placement <= max_digit:
        # declare and initialize empty buckets
        snake_case_ :     list[list]				=	[[] for _ in range(_a    )]
        # split list_of_ints between the buckets
        for i in list_of_ints:
            snake_case_ :     List[Any]				=	int((i / placement) % RADIX    )
            buckets[tmp].append(_a    )
        # put each buckets' contents into list_of_ints
        snake_case_ :     Union[str, Any]				=	0
        for b in range(_a    ):
            for i in buckets[b]:
                snake_case_ :     int				=	i
                a += 1
        # move to next
        placement *= RADIX
    return list_of_ints


if __name__ == "__main__":
     import doctest

     doctest.testmod()










"""simple docstring"""

import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

lowercase__    :  Dict							    =	[
    '''word_embeddings_layernorm.weight''',
    '''word_embeddings_layernorm.bias''',
    '''input_layernorm.weight''',
    '''input_layernorm.bias''',
    '''post_attention_layernorm.weight''',
    '''post_attention_layernorm.bias''',
    '''self_attention.dense.bias''',
    '''mlp.dense_4h_to_h.bias''',
    '''ln_f.weight''',
    '''ln_f.bias''',
]

lowercase__    :  str							    =	[
    '''mlp.dense_4h_to_h.weight''',
    '''self_attention.dense.weight''',
]




def __lowercase			(							_a		,			_a    ):
    snake_case_ :     Optional[int]				=	{
        '''word_embeddings.weight''': '''word_embeddings.weight''',
        '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
        '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
        '''weight''': '''ln_f.weight''',
        '''bias''': '''ln_f.bias''',
    }

    if key in layer_rename_map:
        return layer_rename_map[key]

    # Handle transformer blocks
    snake_case_ :     List[Any]				=	int(re.match(r'''.*layer_(\d*).*'''		,			_a    )[1]    )
    layer_number -= 3
    return f"h.{layer_number}." + key




def __lowercase			(							_a    ):
    if dtype == torch.bool:
        return 1 / 8
    snake_case_ :     Dict				=	re.search(r'''[^\d](\d+)$'''		,			str(_a    )    )
    if bit_search is None:
        raise ValueError(f"`dtype` is not a valid dtype: {dtype}."    )
    snake_case_ :     Optional[int]				=	int(bit_search.groups()[0]    )
    return bit_size // 8




def __lowercase			(							_a		,			_a		,			_a		,			_a		,			_a    ):
    # Construct model
    if bloom_config_file == "":
        snake_case_ :     int				=	BloomConfig()
    else:
        snake_case_ :     List[str]				=	BloomConfig.from_json_file(_a    )

    if shard_model:
        snake_case_ :     List[str]				=	os.listdir(_a    )
        snake_case_ :     int				=	sorted(filter(lambda _a    : s.startswith('''layer'''    ) and "model_00" in s		,			_a    )    )

        snake_case_ :     List[str]				=	{'''weight_map''': {}, '''metadata''': {}}
        snake_case_ :     Any				=	0

        snake_case_ :     Union[str, Any]				=	None

        snake_case_ :     List[str]				=	BloomConfig()

        for j, file in enumerate(_a    ):
            print('''Processing file: {}'''.format(_a    )    )
            snake_case_ :     Dict				=	None

            for i in range(_a    ):
                # load all TP files
                snake_case_ :     Union[str, Any]				=	file.replace('''model_00'''		,			f"model_0{i}"    )
                snake_case_ :     List[str]				=	torch.load(os.path.join(_a		,			_a    )		,			map_location='''cpu'''    )

                # Rename keys in the transformers names
                snake_case_ :     str				=	list(temp.keys()    )
                for key in keys:
                    snake_case_ :     Any				=	temp.pop(_a    )

                if tensors is None:
                    snake_case_ :     Any				=	temp
                else:
                    for key in tensors.keys():
                        if any(key.endswith(_a    ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH    ):
                            # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                            tensors[key] += temp[key]
                        else:
                            # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                            snake_case_ :     Tuple				=	1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN    ) else 0
                            # We concatenate these weights accross TP ranks
                            snake_case_ :     List[str]				=	torch.cat([tensors[key], temp[key]]		,			dim=_a    )

            # Divide by the number of TP the weights we want to average
            for key in tensors.keys():
                if any(key.endswith(_a    ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH    ):
                    snake_case_ :     Any				=	tensors[key] / pretraining_tp
            torch.save(
                _a		,			os.path.join(
                    _a		,			'''pytorch_model_{}-of-{}.bin'''.format(str(j + 1    ).zfill(5    )		,			str(len(_a    )    ).zfill(5    )    )		,			)		,			)

            for key in tensors.keys():
                snake_case_ :     List[str]				=	tensors[key]
                total_size += value.numel() * get_dtype_size(value.dtype    )
                if key not in index_dict["weight_map"]:
                    snake_case_ :     List[str]				=	'''pytorch_model_{}-of-{}.bin'''.format(
                        str(j + 1    ).zfill(5    )		,			str(len(_a    )    ).zfill(5    )    )

        snake_case_ :     int				=	BloomConfig()
        snake_case_ :     Any				=	pytorch_dump_folder_path + '''/''' + CONFIG_NAME
        snake_case_ :     Dict				=	total_size
        with open(_a		,			'''w'''		,			encoding='''utf-8'''    ) as f:
            f.write(config.to_json_string()    )
        with open(os.path.join(_a		,			WEIGHTS_NAME + '''.index.json'''    )		,			'''w'''		,			encoding='''utf-8'''    ) as f:
            snake_case_ :     Tuple				=	json.dumps(_a		,			indent=2		,			sort_keys=_a    ) + '''\n'''
            f.write(_a    )
    else:
        snake_case_ :     Union[str, Any]				=	BloomModel(_a    )

        snake_case_ :     List[str]				=	os.listdir(_a    )
        snake_case_ :     Dict				=	sorted(filter(lambda _a    : s.startswith('''layer'''    ) and "model_00" in s		,			_a    )    )

        snake_case_ :     List[Any]				=	None
        for i, file in enumerate(_a    ):
            snake_case_ :     Optional[Any]				=	None
            for i in range(_a    ):
                # load all TP files
                snake_case_ :     List[str]				=	file.replace('''model_00'''		,			f"model_0{i}"    )
                snake_case_ :     Optional[Any]				=	torch.load(os.path.join(_a		,			_a    )		,			map_location='''cpu'''    )

                # Rename keys in the transformers names
                snake_case_ :     str				=	list(temp.keys()    )
                for key in keys:
                    snake_case_ :     str				=	temp.pop(_a    )

                if tensors is None:
                    snake_case_ :     int				=	temp
                else:
                    for key in tensors.keys():
                        # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                        if any(key.endswith(_a    ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH    ):
                            tensors[key] += temp[key]
                        else:
                            # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                            snake_case_ :     Tuple				=	1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN    ) else 0
                            # We concatenate these weights accross TP ranks
                            snake_case_ :     Optional[Any]				=	torch.cat([tensors[key], temp[key]]		,			dim=_a    )

            # Divide by the number of TP the weights we want to average
            for key in tensors.keys():
                if any(key.endswith(_a    ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH    ):
                    snake_case_ :     Union[str, Any]				=	tensors[key] / pretraining_tp

            snake_case_ :     Any				=	model.load_state_dict(_a		,			strict=_a    )
            assert not other_keys.unexpected_keys, f"The keys {other_keys.unexpected_keys} are unexpected"
            if missing_keys is None:
                snake_case_ :     Optional[int]				=	set(other_keys.missing_keys    )
            else:
                snake_case_ :     Tuple				=	missing_keys.intersection(set(other_keys.missing_keys    )    )

        assert not missing_keys, f"The keys {missing_keys} are missing"

        # Save pytorch-model
        os.makedirs(_a		,			exist_ok=_a    )
        snake_case_ :     List[str]				=	pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
        snake_case_ :     Optional[Any]				=	pytorch_dump_folder_path + '''/''' + CONFIG_NAME
        print(f"Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}"    )
        if config.torch_dtype is not None:
            snake_case_ :     Optional[Any]				=	model.to(config.torch_dtype    )
        torch.save(model.state_dict()		,			_a    )
        print(f"Save configuration file to {pytorch_config_dump_path}"    )
        with open(_a		,			'''w'''		,			encoding='''utf-8'''    ) as f:
            f.write(config.to_json_string()    )


if __name__ == "__main__":
     lowercase__    :  str							    =	argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         '''--bloom_checkpoint_path''',
         default=None,
         type=str,
         required=True,
         help='''Path to the Megatron-LM checkpoint path.''',
     )
     parser.add_argument(
         '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
     )
     parser.add_argument(
         '''--bloom_config_file''',
         default='''''',
         type=str,
         help=(
             '''An optional config json file corresponding to the pre-trained model. \n'''
             '''This specifies the model architecture.'''
         ),
     )
     parser.add_argument(
         '''--shard_model''',
         action='''store_true''',
         help='''An optional setting to shard the output model \nThis enables sharding the converted checkpoint''',
     )
     parser.add_argument(
         '''--pretraining_tp''',
         default=4,
         type=int,
         help='''Pretraining TP rank that has been used when training the model in Megatron-LM \n''',
     )
     lowercase__    :  List[Any]							    =	parser.parse_args()
     convert_bloom_checkpoint_to_pytorch(
         args.bloom_checkpoint_path,
         args.bloom_config_file,
         args.pytorch_dump_folder_path,
         args.shard_model,
         args.pretraining_tp,
     )





