



"""simple docstring"""

# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from packaging import version

from .. import __version__
from .constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD
from .doc import (
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    copy_func,
    replace_return_docstrings,
)
from .generic import (
    ContextManagers,
    ExplicitEnum,
    ModelOutput,
    PaddingStrategy,
    TensorType,
    add_model_info_to_auto_map,
    cached_property,
    can_return_loss,
    expand_dims,
    find_labels,
    flatten_dict,
    infer_framework,
    is_jax_tensor,
    is_numpy_array,
    is_tensor,
    is_tf_symbolic_tensor,
    is_tf_tensor,
    is_torch_device,
    is_torch_dtype,
    is_torch_tensor,
    reshape,
    squeeze,
    strtobool,
    tensor_size,
    to_numpy,
    to_py_obj,
    transpose,
    working_or_temp_dir,
)
from .hub import (
    CLOUDFRONT_DISTRIB_PREFIX,
    DISABLE_TELEMETRY,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_PREFIX,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    S3_BUCKET_PREFIX,
    TRANSFORMERS_CACHE,
    TRANSFORMERS_DYNAMIC_MODULE_NAME,
    EntryNotFoundError,
    PushToHubMixin,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    cached_file,
    default_cache_path,
    define_sagemaker_information,
    download_url,
    extract_commit_hash,
    get_cached_models,
    get_file_from_repo,
    get_full_repo_name,
    has_file,
    http_user_agent,
    is_offline_mode,
    is_remote_url,
    move_cache,
    send_example_telemetry,
    try_to_load_from_cache,
)
from .import_utils import (
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    TORCH_FX_REQUIRED_VERSION,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    DummyObject,
    OptionalDependencyNotAvailable,
    _LazyModule,
    ccl_version,
    direct_transformers_import,
    get_torch_version,
    is_accelerate_available,
    is_apex_available,
    is_bitsandbytes_available,
    is_bsa_available,
    is_coloredlogs_available,
    is_cython_available,
    is_datasets_available,
    is_decord_available,
    is_detectrona_available,
    is_faiss_available,
    is_flax_available,
    is_ftfy_available,
    is_in_notebook,
    is_ipex_available,
    is_jieba_available,
    is_jumanpp_available,
    is_kenlm_available,
    is_keras_nlp_available,
    is_librosa_available,
    is_natten_available,
    is_ninja_available,
    is_onnx_available,
    is_openai_available,
    is_optimum_available,
    is_pandas_available,
    is_peft_available,
    is_phonemizer_available,
    is_protobuf_available,
    is_psutil_available,
    is_pyanvml_available,
    is_pyctcdecode_available,
    is_pytesseract_available,
    is_pytest_available,
    is_pytorch_quantization_available,
    is_rjieba_available,
    is_sacremoses_available,
    is_safetensors_available,
    is_sagemaker_dp_enabled,
    is_sagemaker_mp_enabled,
    is_scipy_available,
    is_sentencepiece_available,
    is_seqio_available,
    is_sklearn_available,
    is_soundfile_availble,
    is_spacy_available,
    is_speech_available,
    is_sudachi_available,
    is_tensorflow_probability_available,
    is_tensorflow_text_available,
    is_tfaonnx_available,
    is_tf_available,
    is_timm_available,
    is_tokenizers_available,
    is_torch_available,
    is_torch_bfaa_available,
    is_torch_bfaa_cpu_available,
    is_torch_bfaa_gpu_available,
    is_torch_compile_available,
    is_torch_cuda_available,
    is_torch_fx_available,
    is_torch_fx_proxy,
    is_torch_mps_available,
    is_torch_neuroncore_available,
    is_torch_tensorrt_fx_available,
    is_torch_tfaa_available,
    is_torch_tpu_available,
    is_torchaudio_available,
    is_torchdistx_available,
    is_torchdynamo_available,
    is_torchvision_available,
    is_training_run_on_sagemaker,
    is_vision_available,
    requires_backends,
    torch_only_method,
)


lowercase__    :  Dict							    =	'''pytorch_model.bin'''
lowercase__    :  Optional[int]							    =	'''pytorch_model.bin.index.json'''
lowercase__    :  Any							    =	'''adapter_config.json'''
lowercase__    :  List[str]							    =	'''adapter_model.bin'''
lowercase__    :  int							    =	'''adapter_model.safetensors'''
lowercase__    :  List[Any]							    =	'''tf_model.h5'''
lowercase__    :  Union[str, Any]							    =	'''tf_model.h5.index.json'''
lowercase__    :  List[str]							    =	'''model.ckpt'''
lowercase__    :  Optional[int]							    =	'''flax_model.msgpack'''
lowercase__    :  Any							    =	'''flax_model.msgpack.index.json'''
lowercase__    :  int							    =	'''model.safetensors'''
lowercase__    :  Dict							    =	'''model.safetensors.index.json'''
lowercase__    :  Optional[int]							    =	'''config.json'''
lowercase__    :  Optional[Any]							    =	'''preprocessor_config.json'''
lowercase__    :  str							    =	FEATURE_EXTRACTOR_NAME
lowercase__    :  Union[str, Any]							    =	'''generation_config.json'''
lowercase__    :  Union[str, Any]							    =	'''modelcard.json'''

lowercase__    :  Optional[int]							    =	'''▁'''
lowercase__    :  List[str]							    =	SENTENCEPIECE_UNDERLINE  # Kept for backward compatibility

lowercase__    :  Tuple							    =	[
    [[0, 1, 0, 1], [1, 0, 0, 1]]
] * 2  # Needs to have 0s and 1s only since XLM uses it for langs too.
lowercase__    :  Optional[Any]							    =	[[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]
lowercase__    :  Optional[int]							    =	[[1, 1, 1, 1, 1], [1, 1, 1, 0, 0], [0, 0, 0, 1, 1]]




def __lowercase			(							_a    ):
    if version.parse(_a    ) < version.parse(_a    ):
        if "dev" in min_version:
            snake_case_ :     Any				=	(
                '''This example requires a source install from HuggingFace Transformers (see '''
                '''`https://huggingface.co/docs/transformers/installation#install-from-source`),'''
            )
        else:
            snake_case_ :     int				=	f"This example requires a minimum version of {min_version},"
        error_message += f" but the version found is {__version__}.\n"
        raise ImportError(
            error_message
            + '''Check out https://github.com/huggingface/transformers/tree/main/examples#important-note for the examples corresponding to other '''
            '''versions of HuggingFace Transformers.'''    )










"""simple docstring"""

import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

lowercase__    :  Dict							    =	[
    '''word_embeddings_layernorm.weight''',
    '''word_embeddings_layernorm.bias''',
    '''input_layernorm.weight''',
    '''input_layernorm.bias''',
    '''post_attention_layernorm.weight''',
    '''post_attention_layernorm.bias''',
    '''self_attention.dense.bias''',
    '''mlp.dense_4h_to_h.bias''',
    '''ln_f.weight''',
    '''ln_f.bias''',
]

lowercase__    :  str							    =	[
    '''mlp.dense_4h_to_h.weight''',
    '''self_attention.dense.weight''',
]




def __lowercase			(							_a		,			_a    ):
    snake_case_ :     Optional[int]				=	{
        '''word_embeddings.weight''': '''word_embeddings.weight''',
        '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
        '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
        '''weight''': '''ln_f.weight''',
        '''bias''': '''ln_f.bias''',
    }

    if key in layer_rename_map:
        return layer_rename_map[key]

    # Handle transformer blocks
    snake_case_ :     List[Any]				=	int(re.match(r'''.*layer_(\d*).*'''		,			_a    )[1]    )
    layer_number -= 3
    return f"h.{layer_number}." + key




def __lowercase			(							_a    ):
    if dtype == torch.bool:
        return 1 / 8
    snake_case_ :     Dict				=	re.search(r'''[^\d](\d+)$'''		,			str(_a    )    )
    if bit_search is None:
        raise ValueError(f"`dtype` is not a valid dtype: {dtype}."    )
    snake_case_ :     Optional[int]				=	int(bit_search.groups()[0]    )
    return bit_size // 8




def __lowercase			(							_a		,			_a		,			_a		,			_a		,			_a    ):
    # Construct model
    if bloom_config_file == "":
        snake_case_ :     int				=	BloomConfig()
    else:
        snake_case_ :     List[str]				=	BloomConfig.from_json_file(_a    )

    if shard_model:
        snake_case_ :     List[str]				=	os.listdir(_a    )
        snake_case_ :     int				=	sorted(filter(lambda _a    : s.startswith('''layer'''    ) and "model_00" in s		,			_a    )    )

        snake_case_ :     List[str]				=	{'''weight_map''': {}, '''metadata''': {}}
        snake_case_ :     Any				=	0

        snake_case_ :     Union[str, Any]				=	None

        snake_case_ :     List[str]				=	BloomConfig()

        for j, file in enumerate(_a    ):
            print('''Processing file: {}'''.format(_a    )    )
            snake_case_ :     Dict				=	None

            for i in range(_a    ):
                # load all TP files
                snake_case_ :     Union[str, Any]				=	file.replace('''model_00'''		,			f"model_0{i}"    )
                snake_case_ :     List[str]				=	torch.load(os.path.join(_a		,			_a    )		,			map_location='''cpu'''    )

                # Rename keys in the transformers names
                snake_case_ :     str				=	list(temp.keys()    )
                for key in keys:
                    snake_case_ :     Any				=	temp.pop(_a    )

                if tensors is None:
                    snake_case_ :     Any				=	temp
                else:
                    for key in tensors.keys():
                        if any(key.endswith(_a    ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH    ):
                            # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                            tensors[key] += temp[key]
                        else:
                            # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                            snake_case_ :     Tuple				=	1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN    ) else 0
                            # We concatenate these weights accross TP ranks
                            snake_case_ :     List[str]				=	torch.cat([tensors[key], temp[key]]		,			dim=_a    )

            # Divide by the number of TP the weights we want to average
            for key in tensors.keys():
                if any(key.endswith(_a    ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH    ):
                    snake_case_ :     Any				=	tensors[key] / pretraining_tp
            torch.save(
                _a		,			os.path.join(
                    _a		,			'''pytorch_model_{}-of-{}.bin'''.format(str(j + 1    ).zfill(5    )		,			str(len(_a    )    ).zfill(5    )    )		,			)		,			)

            for key in tensors.keys():
                snake_case_ :     List[str]				=	tensors[key]
                total_size += value.numel() * get_dtype_size(value.dtype    )
                if key not in index_dict["weight_map"]:
                    snake_case_ :     List[str]				=	'''pytorch_model_{}-of-{}.bin'''.format(
                        str(j + 1    ).zfill(5    )		,			str(len(_a    )    ).zfill(5    )    )

        snake_case_ :     int				=	BloomConfig()
        snake_case_ :     Any				=	pytorch_dump_folder_path + '''/''' + CONFIG_NAME
        snake_case_ :     Dict				=	total_size
        with open(_a		,			'''w'''		,			encoding='''utf-8'''    ) as f:
            f.write(config.to_json_string()    )
        with open(os.path.join(_a		,			WEIGHTS_NAME + '''.index.json'''    )		,			'''w'''		,			encoding='''utf-8'''    ) as f:
            snake_case_ :     Tuple				=	json.dumps(_a		,			indent=2		,			sort_keys=_a    ) + '''\n'''
            f.write(_a    )
    else:
        snake_case_ :     Union[str, Any]				=	BloomModel(_a    )

        snake_case_ :     List[str]				=	os.listdir(_a    )
        snake_case_ :     Dict				=	sorted(filter(lambda _a    : s.startswith('''layer'''    ) and "model_00" in s		,			_a    )    )

        snake_case_ :     List[Any]				=	None
        for i, file in enumerate(_a    ):
            snake_case_ :     Optional[Any]				=	None
            for i in range(_a    ):
                # load all TP files
                snake_case_ :     List[str]				=	file.replace('''model_00'''		,			f"model_0{i}"    )
                snake_case_ :     Optional[Any]				=	torch.load(os.path.join(_a		,			_a    )		,			map_location='''cpu'''    )

                # Rename keys in the transformers names
                snake_case_ :     str				=	list(temp.keys()    )
                for key in keys:
                    snake_case_ :     str				=	temp.pop(_a    )

                if tensors is None:
                    snake_case_ :     int				=	temp
                else:
                    for key in tensors.keys():
                        # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                        if any(key.endswith(_a    ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH    ):
                            tensors[key] += temp[key]
                        else:
                            # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                            snake_case_ :     Tuple				=	1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN    ) else 0
                            # We concatenate these weights accross TP ranks
                            snake_case_ :     Optional[Any]				=	torch.cat([tensors[key], temp[key]]		,			dim=_a    )

            # Divide by the number of TP the weights we want to average
            for key in tensors.keys():
                if any(key.endswith(_a    ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH    ):
                    snake_case_ :     Union[str, Any]				=	tensors[key] / pretraining_tp

            snake_case_ :     Any				=	model.load_state_dict(_a		,			strict=_a    )
            assert not other_keys.unexpected_keys, f"The keys {other_keys.unexpected_keys} are unexpected"
            if missing_keys is None:
                snake_case_ :     Optional[int]				=	set(other_keys.missing_keys    )
            else:
                snake_case_ :     Tuple				=	missing_keys.intersection(set(other_keys.missing_keys    )    )

        assert not missing_keys, f"The keys {missing_keys} are missing"

        # Save pytorch-model
        os.makedirs(_a		,			exist_ok=_a    )
        snake_case_ :     List[str]				=	pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
        snake_case_ :     Optional[Any]				=	pytorch_dump_folder_path + '''/''' + CONFIG_NAME
        print(f"Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}"    )
        if config.torch_dtype is not None:
            snake_case_ :     Optional[Any]				=	model.to(config.torch_dtype    )
        torch.save(model.state_dict()		,			_a    )
        print(f"Save configuration file to {pytorch_config_dump_path}"    )
        with open(_a		,			'''w'''		,			encoding='''utf-8'''    ) as f:
            f.write(config.to_json_string()    )


if __name__ == "__main__":
     lowercase__    :  str							    =	argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         '''--bloom_checkpoint_path''',
         default=None,
         type=str,
         required=True,
         help='''Path to the Megatron-LM checkpoint path.''',
     )
     parser.add_argument(
         '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
     )
     parser.add_argument(
         '''--bloom_config_file''',
         default='''''',
         type=str,
         help=(
             '''An optional config json file corresponding to the pre-trained model. \n'''
             '''This specifies the model architecture.'''
         ),
     )
     parser.add_argument(
         '''--shard_model''',
         action='''store_true''',
         help='''An optional setting to shard the output model \nThis enables sharding the converted checkpoint''',
     )
     parser.add_argument(
         '''--pretraining_tp''',
         default=4,
         type=int,
         help='''Pretraining TP rank that has been used when training the model in Megatron-LM \n''',
     )
     lowercase__    :  List[Any]							    =	parser.parse_args()
     convert_bloom_checkpoint_to_pytorch(
         args.bloom_checkpoint_path,
         args.bloom_config_file,
         args.pytorch_dump_folder_path,
         args.shard_model,
         args.pretraining_tp,
     )





