

import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

_UpperCamelCase 			=		[
    "word_embeddings_layernorm.weight",
    "word_embeddings_layernorm.bias",
    "input_layernorm.weight",
    "input_layernorm.bias",
    "post_attention_layernorm.weight",
    "post_attention_layernorm.bias",
    "self_attention.dense.bias",
    "mlp.dense_4h_to_h.bias",
    "ln_f.weight",
    "ln_f.bias",
]

_UpperCamelCase 			=		[
    "mlp.dense_4h_to_h.weight",
    "self_attention.dense.weight",
]







def 		_lowercase (     lowercase__   ,	lowercase__						):
      __lowerCAmelCase					:						Any					=					{
          '''word_embeddings.weight''': '''word_embeddings.weight''',
          '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
          '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
          '''weight''': '''ln_f.weight''',
          '''bias''': '''ln_f.bias''',
      }

      if key in layer_rename_map:
            return layer_rename_map[key]

      # Handle transformer blocks
      __lowerCAmelCase					:						str					=					int(re.match(r'''.*layer_(\d*).*'''   ,	lowercase__						)[1]						)
      layer_number -= 3
      return f"""h.{layer_number}.""" + key







def 		_lowercase (     lowercase__						):
      if dtype == torch.bool:
            return 1 / 8
      __lowerCAmelCase					:						Dict					=					re.search(r'''[^\d](\d+)$'''   ,	str(lowercase__						)						)
      if bit_search is None:
            raise ValueError(f"""`dtype` is not a valid dtype: {dtype}."""						)
      __lowerCAmelCase					:						Dict					=					int(bit_search.groups()[0]						)
      return bit_size // 8







def 		_lowercase (     lowercase__   ,	lowercase__   ,	lowercase__   ,	lowercase__   ,	lowercase__						):
      # Construct model
      if bloom_config_file == "":
            __lowerCAmelCase					:						Tuple					=					BloomConfig()
      else:
            __lowerCAmelCase					:						List[Any]					=					BloomConfig.from_json_file(lowercase__						)

      if shard_model:
            __lowerCAmelCase					:						Union[str, Any]					=					os.listdir(lowercase__						)
            __lowerCAmelCase					:						Optional[int]					=					sorted(filter(lambda lowercase__						: s.startswith('''layer'''						) and "model_00" in s   ,	lowercase__						)						)

            __lowerCAmelCase					:						str					=					{'''weight_map''': {}, '''metadata''': {}}
            __lowerCAmelCase					:						Optional[Any]					=					0

            __lowerCAmelCase					:						int					=					None

            __lowerCAmelCase					:						List[Any]					=					BloomConfig()

            for j, file in enumerate(lowercase__						):
                  print('''Processing file: {}'''.format(lowercase__						)						)
                  __lowerCAmelCase					:						Union[str, Any]					=					None

                  for i in range(lowercase__						):
                        # load all TP files
                        __lowerCAmelCase					:						str					=					file.replace('''model_00'''   ,	f"""model_0{i}"""						)
                        __lowerCAmelCase					:						Tuple					=					torch.load(os.path.join(lowercase__   ,	lowercase__						)   ,	map_location='''cpu'''						)

                        # Rename keys in the transformers names
                        __lowerCAmelCase					:						int					=					list(temp.keys()						)
                        for key in keys:
                              __lowerCAmelCase					:						List[str]					=					temp.pop(lowercase__						)

                        if tensors is None:
                              __lowerCAmelCase					:						List[Any]					=					temp
                        else:
                              for key in tensors.keys():
                                    if any(key.endswith(lowercase__						) for end in WEIGHTS_TO_AVERAGE_ENDSWITH						):
                                          # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                                          tensors[key] += temp[key]
                                    else:
                                          # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                                          __lowerCAmelCase					:						Optional[Any]					=					1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN						) else 0
                                          # We concatenate these weights accross TP ranks
                                          __lowerCAmelCase					:						int					=					torch.cat([tensors[key], temp[key]]   ,	dim=lowercase__						)

            # Divide by the number of TP the weights we want to average
                  for key in tensors.keys():
                        if any(key.endswith(lowercase__						) for end in WEIGHTS_TO_AVERAGE_ENDSWITH						):
                              __lowerCAmelCase					:						int					=					tensors[key] / pretraining_tp
                  torch.save(
                      lowercase__   ,	os.path.join(
                          lowercase__   ,	'''pytorch_model_{}-of-{}.bin'''.format(str(j + 1						).zfill(5						)   ,	str(len(lowercase__						)						).zfill(5						)						)   ,	)   ,	)

                  for key in tensors.keys():
                        __lowerCAmelCase					:						Union[str, Any]					=					tensors[key]
                        total_size += value.numel() * get_dtype_size(value.dtype						)
                        if key not in index_dict["weight_map"]:
                              __lowerCAmelCase					:						int					=					'''pytorch_model_{}-of-{}.bin'''.format(
                                  str(j + 1						).zfill(5						)   ,	str(len(lowercase__						)						).zfill(5						)						)

            __lowerCAmelCase					:						Optional[int]					=					BloomConfig()
            __lowerCAmelCase					:						Optional[Any]					=					pytorch_dump_folder_path + '''/''' + CONFIG_NAME
            __lowerCAmelCase					:						int					=					total_size
            with open(lowercase__   ,	'''w'''   ,	encoding='''utf-8'''						) as f:
                  f.write(config.to_json_string()						)
            with open(os.path.join(lowercase__   ,	WEIGHTS_NAME + '''.index.json'''						)   ,	'''w'''   ,	encoding='''utf-8'''						) as f:
                  __lowerCAmelCase					:						int					=					json.dumps(lowercase__   ,	indent=2   ,	sort_keys=lowercase__						) + '''\n'''
                  f.write(lowercase__						)
      else:
            __lowerCAmelCase					:						Dict					=					BloomModel(lowercase__						)

            __lowerCAmelCase					:						Optional[Any]					=					os.listdir(lowercase__						)
            __lowerCAmelCase					:						Dict					=					sorted(filter(lambda lowercase__						: s.startswith('''layer'''						) and "model_00" in s   ,	lowercase__						)						)

            __lowerCAmelCase					:						Union[str, Any]					=					None
            for i, file in enumerate(lowercase__						):
                  __lowerCAmelCase					:						List[Any]					=					None
                  for i in range(lowercase__						):
                        # load all TP files
                        __lowerCAmelCase					:						Tuple					=					file.replace('''model_00'''   ,	f"""model_0{i}"""						)
                        __lowerCAmelCase					:						str					=					torch.load(os.path.join(lowercase__   ,	lowercase__						)   ,	map_location='''cpu'''						)

                        # Rename keys in the transformers names
                        __lowerCAmelCase					:						Optional[Any]					=					list(temp.keys()						)
                        for key in keys:
                              __lowerCAmelCase					:						List[Any]					=					temp.pop(lowercase__						)

                        if tensors is None:
                              __lowerCAmelCase					:						List[Any]					=					temp
                        else:
                              for key in tensors.keys():
                                    # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                                    if any(key.endswith(lowercase__						) for end in WEIGHTS_TO_AVERAGE_ENDSWITH						):
                                          tensors[key] += temp[key]
                                    else:
                                          # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                                          __lowerCAmelCase					:						Dict					=					1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN						) else 0
                                          # We concatenate these weights accross TP ranks
                                          __lowerCAmelCase					:						List[str]					=					torch.cat([tensors[key], temp[key]]   ,	dim=lowercase__						)

            # Divide by the number of TP the weights we want to average
                  for key in tensors.keys():
                        if any(key.endswith(lowercase__						) for end in WEIGHTS_TO_AVERAGE_ENDSWITH						):
                              __lowerCAmelCase					:						List[Any]					=					tensors[key] / pretraining_tp

                  __lowerCAmelCase					:						Optional[int]					=					model.load_state_dict(lowercase__   ,	strict=lowercase__						)
                  assert not other_keys.unexpected_keys, f"""The keys {other_keys.unexpected_keys} are unexpected"""
                  if missing_keys is None:
                        __lowerCAmelCase					:						Dict					=					set(other_keys.missing_keys						)
                  else:
                        __lowerCAmelCase					:						List[Any]					=					missing_keys.intersection(set(other_keys.missing_keys						)						)

            assert not missing_keys, f"""The keys {missing_keys} are missing"""

            # Save pytorch-model
            os.makedirs(lowercase__   ,	exist_ok=lowercase__						)
            __lowerCAmelCase					:						Any					=					pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
            __lowerCAmelCase					:						str					=					pytorch_dump_folder_path + '''/''' + CONFIG_NAME
            print(f"""Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}"""						)
            if config.torch_dtype is not None:
                  __lowerCAmelCase					:						Union[str, Any]					=					model.to(config.torch_dtype						)
            torch.save(model.state_dict()   ,	lowercase__						)
            print(f"""Save configuration file to {pytorch_config_dump_path}"""						)
            with open(lowercase__   ,	'''w'''   ,	encoding='''utf-8'''						) as f:
                  f.write(config.to_json_string()						)


if __name__ == "__main__":
 _UpperCamelCase 			=		argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     "--bloom_checkpoint_path",
     default=None,
     type=str,
     required=True,
     help="Path to the Megatron-LM checkpoint path.",
 )
 parser.add_argument(
     "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
 )
 parser.add_argument(
     "--bloom_config_file",
     default="",
     type=str,
     help=(
         "An optional config json file corresponding to the pre-trained model. \n"
         "This specifies the model architecture."
     ),
 )
 parser.add_argument(
     "--shard_model",
     action="store_true",
     help="An optional setting to shard the output model \nThis enables sharding the converted checkpoint",
 )
 parser.add_argument(
     "--pretraining_tp",
     default=4,
     type=int,
     help="Pretraining TP rank that has been used when training the model in Megatron-LM \n",
 )
 _UpperCamelCase 			=		parser.parse_args()
 convert_bloom_checkpoint_to_pytorch(
     args.bloom_checkpoint_path,
     args.bloom_config_file,
     args.pytorch_dump_folder_path,
     args.shard_model,
     args.pretraining_tp,
 )





def 		_lowercase (     lowercase__   ,	lowercase__						):
      return (pointa[0] - pointa[0]) ** 2 + (pointa[1] - pointa[1]) ** 2







def 		_lowercase (     lowercase__   ,	lowercase__=0						):
      return sorted(lowercase__   ,	key=lambda lowercase__						: x[column]						)







def 		_lowercase (     lowercase__   ,	lowercase__   ,	lowercase__=float('''inf'''						)						):

      for i in range(points_counts - 1						):
            for j in range(i + 1   ,	lowercase__						):
                  __lowerCAmelCase					:						List[str]					=					euclidean_distance_sqr(points[i]   ,	points[j]						)
                  if current_dis < min_dis:
                        __lowerCAmelCase					:						Tuple					=					current_dis
      return min_dis







def 		_lowercase (     lowercase__   ,	lowercase__   ,	lowercase__=float('''inf'''						)						):

      for i in range(min(6   ,	points_counts - 1						)   ,	lowercase__						):
            for j in range(max(0   ,	i - 6						)   ,	lowercase__						):
                  __lowerCAmelCase					:						Union[str, Any]					=					euclidean_distance_sqr(points[i]   ,	points[j]						)
                  if current_dis < min_dis:
                        __lowerCAmelCase					:						int					=					current_dis
      return min_dis







def 		_lowercase (     lowercase__   ,	lowercase__   ,	lowercase__						):

      # base case
      if points_counts <= 3:
            return dis_between_closest_pair(lowercase__   ,	lowercase__						)

      # recursion
      __lowerCAmelCase					:						Optional[Any]					=					points_counts // 2
      __lowerCAmelCase					:						Optional[Any]					=					closest_pair_of_points_sqr(
          lowercase__   ,	points_sorted_on_y[:mid]   ,	lowercase__						)
      __lowerCAmelCase					:						str					=					closest_pair_of_points_sqr(
          lowercase__   ,	points_sorted_on_y[mid:]   ,	points_counts - mid						)
      __lowerCAmelCase					:						Optional[int]					=					min(lowercase__   ,	lowercase__						)
      __lowerCAmelCase					:						Tuple					=					[]
      for point in points_sorted_on_x:
            if abs(point[0] - points_sorted_on_x[mid][0]						) < closest_pair_dis:
                  cross_strip.append(lowercase__						)

      __lowerCAmelCase					:						List[Any]					=					dis_between_closest_in_strip(
          lowercase__   ,	len(lowercase__						)   ,	lowercase__						)
      return min(lowercase__   ,	lowercase__						)







def 		_lowercase (     lowercase__   ,	lowercase__						):
      __lowerCAmelCase					:						Union[str, Any]					=					column_based_sort(lowercase__   ,	column=0						)
      __lowerCAmelCase					:						Any					=					column_based_sort(lowercase__   ,	column=1						)
      return (
          closest_pair_of_points_sqr(
              lowercase__   ,	lowercase__   ,	lowercase__						)
      ) ** 0.5


if __name__ == "__main__":
 _UpperCamelCase 			=		[(2, 3), (12, 30), (40, 50), (5, 1), (12, 10), (3, 4)]
 print("Distance:", closest_pair_of_points(points, len(points)))


