
"""simple docstring"""







import argparse
import glob
import logging
import os
import time
from argparse import Namespace

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from torch.utils.data import DataLoader, TensorDataset

from transformers import glue_compute_metrics as compute_metrics
from transformers import glue_convert_examples_to_features as convert_examples_to_features
from transformers import glue_output_modes, glue_tasks_num_labels
from transformers import glue_processors as processors


lowerCamelCase__   					=		logging.getLogger(__name__)




class  A__       (	_lowerCamelCase):
		A_						:       str				   =	'sequence-classification'

		def __init__(      self       ,		_SCREAMING_SNAKE_CASE ):
			if type(_SCREAMING_SNAKE_CASE ) == dict:
				__lowerCAmelCase			:						Dict									=      Namespace(**_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			:						Union[str, Any]									=      glue_output_modes[hparams.task]
			__lowerCAmelCase			:						Optional[Any]									=      glue_tasks_num_labels[hparams.task]

			super().__init__(_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE       ,		self.mode )

		def 							__lowerCamelCase							(      self       ,		**_SCREAMING_SNAKE_CASE ):
			return self.model(**_SCREAMING_SNAKE_CASE )

		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE ):
			__lowerCAmelCase			:						Optional[int]									=      {'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3]}

			if self.config.model_type not in ["distilbert", "bart"]:
				__lowerCAmelCase			:						List[str]									=      batch[2] if self.config.model_type in ['bert', 'xlnet', 'albert'] else None

			__lowerCAmelCase			:						Dict									=      self(**_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			:						int									=      outputs[0]

			__lowerCAmelCase			:						Union[str, Any]									=      self.trainer.lr_schedulers[0]['scheduler']
			__lowerCAmelCase			:						Optional[Any]									=      {'loss': loss, 'rate': lr_scheduler.get_last_lr()[-1]}
			return {"loss": loss, "log": tensorboard_logs}

		def 							__lowerCamelCase							(      self ):
			__lowerCAmelCase			:						str									=      self.hparams
			__lowerCAmelCase			:						int									=      processors[args.task]()
			__lowerCAmelCase			:						Any									=      processor.get_labels()

			for mode in ["train", "dev"]:
				__lowerCAmelCase			:						str									=      self._feature_file(_SCREAMING_SNAKE_CASE )
				if os.path.exists(_SCREAMING_SNAKE_CASE ) and not args.overwrite_cache:
					logger.info('Loading features from cached file %s'       ,		_SCREAMING_SNAKE_CASE )
				else:
					logger.info('Creating features from dataset file at %s'       ,		args.data_dir )
					__lowerCAmelCase			:						Dict									=      (
					    processor.get_dev_examples(args.data_dir )
					    if mode == 'dev'
					    else processor.get_train_examples(args.data_dir )
					)
					__lowerCAmelCase			:						List[str]									=      convert_examples_to_features(
					    _SCREAMING_SNAKE_CASE       ,		self.tokenizer       ,		max_length=args.max_seq_length       ,		label_list=self.labels       ,		output_mode=args.glue_output_mode       ,		)
					logger.info('Saving features into cached file %s'       ,		_SCREAMING_SNAKE_CASE )
					torch.save(_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE )

		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE = False ):
			__lowerCAmelCase			:						int									=      'dev' if mode == 'test' else mode

			__lowerCAmelCase			:						Any									=      self._feature_file(_SCREAMING_SNAKE_CASE )
			logger.info('Loading features from cached file %s'       ,		_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			:						Union[str, Any]									=      torch.load(_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			:						Tuple									=      torch.tensor([f.input_ids for f in features]       ,		dtype=torch.long )
			__lowerCAmelCase			:						Tuple									=      torch.tensor([f.attention_mask for f in features]       ,		dtype=torch.long )
			__lowerCAmelCase			:						Union[str, Any]									=      torch.tensor([f.token_type_ids for f in features]       ,		dtype=torch.long )
			if self.hparams.glue_output_mode == "classification":
				__lowerCAmelCase			:						Any									=      torch.tensor([f.label for f in features]       ,		dtype=torch.long )
			elif self.hparams.glue_output_mode == "regression":
				__lowerCAmelCase			:						int									=      torch.tensor([f.label for f in features]       ,		dtype=torch.float )

			return DataLoader(
			    TensorDataset(_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE )       ,		batch_size=_SCREAMING_SNAKE_CASE       ,		shuffle=_SCREAMING_SNAKE_CASE       ,		)

		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE ):
			__lowerCAmelCase			:						Dict									=      {'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3]}

			if self.config.model_type not in ["distilbert", "bart"]:
				__lowerCAmelCase			:						str									=      batch[2] if self.config.model_type in ['bert', 'xlnet', 'albert'] else None

			__lowerCAmelCase			:						List[Any]									=      self(**_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			,   __lowerCAmelCase			:						Union[str, Any]									=      outputs[:2]
			__lowerCAmelCase			:						int									=      logits.detach().cpu().numpy()
			__lowerCAmelCase			:						Any									=      inputs['labels'].detach().cpu().numpy()

			return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}

		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE ):
			__lowerCAmelCase			:						int									=      torch.stack([x['val_loss'] for x in outputs] ).mean().detach().cpu().item()
			__lowerCAmelCase			:						str									=      np.concatenate([x['pred'] for x in outputs]       ,		axis=0 )

			if self.hparams.glue_output_mode == "classification":
				__lowerCAmelCase			:						Tuple									=      np.argmax(_SCREAMING_SNAKE_CASE       ,		axis=1 )
			elif self.hparams.glue_output_mode == "regression":
				__lowerCAmelCase			:						List[str]									=      np.squeeze(_SCREAMING_SNAKE_CASE )

			__lowerCAmelCase			:						Optional[Any]									=      np.concatenate([x['target'] for x in outputs]       ,		axis=0 )
			__lowerCAmelCase			:						str									=      [[] for _ in range(out_label_ids.shape[0] )]
			__lowerCAmelCase			:						Union[str, Any]									=      [[] for _ in range(out_label_ids.shape[0] )]

			__lowerCAmelCase			:						List[Any]									=      {**{'val_loss': val_loss_mean}, **compute_metrics(self.hparams.task       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE )}

			__lowerCAmelCase			:						str									=      dict(results.items() )
			__lowerCAmelCase			:						List[Any]									=      results
			return ret, preds_list, out_label_list

		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE ):
			__lowerCAmelCase			,   __lowerCAmelCase			,   __lowerCAmelCase			:						Dict									=      self._eval_end(_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			:						Union[str, Any]									=      ret['log']
			return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}

		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE ):
			__lowerCAmelCase			,   __lowerCAmelCase			,   __lowerCAmelCase			:						Union[str, Any]									=      self._eval_end(_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			:						Tuple									=      ret['log']
			# `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
			return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}







		@staticmethod
		def 							__lowerCamelCase							(      _SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE ):
			BaseTransformer.add_model_specific_args(_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE )
			parser.add_argument(
			    '--max_seq_length'       ,		default=1_28       ,		type=_SCREAMING_SNAKE_CASE       ,		help=(
			        'The maximum total input sequence length after tokenization. Sequences longer '
			        'than this will be truncated, sequences shorter will be padded.'
			    )       ,		)

			parser.add_argument(
			    '--task'       ,		default=''       ,		type=_SCREAMING_SNAKE_CASE       ,		required=_SCREAMING_SNAKE_CASE       ,		help='The GLUE task to run'       ,		)
			parser.add_argument(
			    '--gpus'       ,		default=0       ,		type=_SCREAMING_SNAKE_CASE       ,		help='The number of GPUs allocated for this, it is by default 0 meaning none'       ,		)

			parser.add_argument(
			    '--overwrite_cache'       ,		action='store_true'       ,		help='Overwrite the cached training and evaluation sets' )

			return parser



def     __lowerCAmelCase				():
	__lowerCAmelCase			:						List[Any]									=      argparse.ArgumentParser()
	add_generic_args(_UpperCamelCase       ,  os.getcwd()		)
	__lowerCAmelCase			:						List[str]									=      GLUETransformer.add_model_specific_args(_UpperCamelCase       ,  os.getcwd()		)
	__lowerCAmelCase			:						Tuple									=      parser.parse_args()

	# If output_dir not provided, a folder will be generated in pwd
	if args.output_dir is None:
		__lowerCAmelCase			:						Dict									=      os.path.join(
		    './results'       ,  F"{args.task}_{time.strftime('%Y%m%d_%H%M%S'		)}"       ,  )
		os.makedirs(args.output_dir		)

	__lowerCAmelCase			:						List[str]									=      GLUETransformer(_UpperCamelCase		)
	__lowerCAmelCase			:						Optional[Any]									=      generic_train(_UpperCamelCase       ,  _UpperCamelCase		)

	# Optionally, predict on dev set and write to output_dir
	if args.do_predict:
		__lowerCAmelCase			:						List[str]									=      sorted(glob.glob(os.path.join(args.output_dir       ,  'checkpoint-epoch=*.ckpt'		)       ,  recursive=_UpperCamelCase		)		)
		__lowerCAmelCase			:						Dict									=      model.load_from_checkpoint(checkpoints[-1]		)
		return trainer.test(_UpperCamelCase		)


if __name__ == "__main__":
							main()



'''simple docstring'''






import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

__A					:  Any											=      [
    "word_embeddings_layernorm.weight",
    "word_embeddings_layernorm.bias",
    "input_layernorm.weight",
    "input_layernorm.bias",
    "post_attention_layernorm.weight",
    "post_attention_layernorm.bias",
    "self_attention.dense.bias",
    "mlp.dense_4h_to_h.bias",
    "ln_f.weight",
    "ln_f.bias",
]

__A					:  Any											=      [
    "mlp.dense_4h_to_h.weight",
    "self_attention.dense.weight",
]







def 				UpperCamelCase_			(    A__					:   str							,    A__					:   str					):
  '''simple docstring'''





  lowerCAmelCase_			:      Dict			=   {
      """word_embeddings.weight""": """word_embeddings.weight""",
      """word_embeddings.norm.weight""": """word_embeddings_layernorm.weight""",
      """word_embeddings.norm.bias""": """word_embeddings_layernorm.bias""",
      """weight""": """ln_f.weight""",
      """bias""": """ln_f.bias""",
  }

  if key in layer_rename_map:
    return layer_rename_map[key]

  # Handle transformer blocks
  lowerCAmelCase_			:      List[str]			=   int(re.match(R""".*layer_(\d*).*"""							,    A__					)[1]					)
  layer_number -= 3
  return f'h.{layer_number}.' + key







def 				UpperCamelCase_			(    A__					:   List[str]					):
  '''simple docstring'''





  if dtype == torch.bool:
    return 1 / 8
  lowerCAmelCase_			:      List[str]			=   re.search(R"""[^\d](\d+)$"""							,    str(A__					)					)
  if bit_search is None:
    raise ValueError(f'`dtype` is not a valid dtype: {dtype}.'					)
  lowerCAmelCase_			:      Dict			=   int(bit_search.groups()[0]					)
  return bit_size // 8







def 				UpperCamelCase_			(    A__					:   int							,    A__					:   Tuple							,    A__					:   Any							,    A__					:   Tuple							,    A__					:   List[str]					):
  '''simple docstring'''





  if bloom_config_file == "":
    lowerCAmelCase_			:      Any			=   BloomConfig()
  else:
    lowerCAmelCase_			:      Any			=   BloomConfig.from_json_file(A__					)

  if shard_model:
    lowerCAmelCase_			:      Any			=   os.listdir(A__					)
    lowerCAmelCase_			:      Optional[int]			=   sorted(filter(lambda A__					: s.startswith("""layer"""					) and "model_00" in s							,    A__					)					)

    lowerCAmelCase_			:      Optional[Any]			=   {"""weight_map""": {}, """metadata""": {}}
    lowerCAmelCase_			:      Optional[Any]			=   0

    lowerCAmelCase_			:      List[str]			=   None

    lowerCAmelCase_			:      Optional[int]			=   BloomConfig()

    for j, file in enumerate(A__					):
      print("""Processing file: {}""".format(A__					)					)
      lowerCAmelCase_			:      List[str]			=   None

      for i in range(A__					):
        # load all TP files
        lowerCAmelCase_			:      Optional[Any]			=   file.replace("""model_00"""							,    f'model_0{i}'					)
        lowerCAmelCase_			:      Optional[int]			=   torch.load(os.path.join(A__							,    A__					)							,    map_location="""cpu"""					)

        # Rename keys in the transformers names
        lowerCAmelCase_			:      str			=   list(temp.keys()					)
        for key in keys:
          lowerCAmelCase_			:      Optional[int]			=   temp.pop(A__					)

        if tensors is None:
          lowerCAmelCase_			:      str			=   temp
        else:
          for key in tensors.keys():
            if any(key.endswith(A__					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
              # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
              tensors[key] += temp[key]
            else:
              # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
              lowerCAmelCase_			:      int			=   1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN					) else 0
              # We concatenate these weights accross TP ranks
              lowerCAmelCase_			:      List[str]			=   torch.cat([tensors[key], temp[key]]							,    dim=A__					)

            # Divide by the number of TP the weights we want to average
      for key in tensors.keys():
        if any(key.endswith(A__					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
          lowerCAmelCase_			:      Optional[int]			=   tensors[key] / pretraining_tp
      torch.save(
          A__							,    os.path.join(
              A__							,    """pytorch_model_{}-of-{}.bin""".format(str(j + 1					).zfill(5					)							,    str(len(A__					)					).zfill(5					)					)							,    )							,    )

      for key in tensors.keys():
        lowerCAmelCase_			:      str			=   tensors[key]
        total_size += value.numel() * get_dtype_size(value.dtype					)
        if key not in index_dict["weight_map"]:
          lowerCAmelCase_			:      Tuple			=   """pytorch_model_{}-of-{}.bin""".format(
              str(j + 1					).zfill(5					)							,    str(len(A__					)					).zfill(5					)					)

    lowerCAmelCase_			:      str			=   BloomConfig()
    lowerCAmelCase_			:      List[Any]			=   pytorch_dump_folder_path + """/""" + CONFIG_NAME
    lowerCAmelCase_			:      List[Any]			=   total_size
    with open(A__							,    """w"""							,    encoding="""utf-8"""					) as f:
      f.write(config.to_json_string()					)
    with open(os.path.join(A__							,    WEIGHTS_NAME + """.index.json"""					)							,    """w"""							,    encoding="""utf-8"""					) as f:
      lowerCAmelCase_			:      Optional[int]			=   json.dumps(A__							,    indent=2							,    sort_keys=A__					) + """\n"""
      f.write(A__					)
  else:
    lowerCAmelCase_			:      int			=   BloomModel(A__					)

    lowerCAmelCase_			:      Union[str, Any]			=   os.listdir(A__					)
    lowerCAmelCase_			:      Tuple			=   sorted(filter(lambda A__					: s.startswith("""layer"""					) and "model_00" in s							,    A__					)					)

    lowerCAmelCase_			:      List[Any]			=   None
    for i, file in enumerate(A__					):
      lowerCAmelCase_			:      List[Any]			=   None
      for i in range(A__					):
        # load all TP files
        lowerCAmelCase_			:      str			=   file.replace("""model_00"""							,    f'model_0{i}'					)
        lowerCAmelCase_			:      Optional[Any]			=   torch.load(os.path.join(A__							,    A__					)							,    map_location="""cpu"""					)

        # Rename keys in the transformers names
        lowerCAmelCase_			:      Union[str, Any]			=   list(temp.keys()					)
        for key in keys:
          lowerCAmelCase_			:      str			=   temp.pop(A__					)

        if tensors is None:
          lowerCAmelCase_			:      int			=   temp
        else:
          for key in tensors.keys():
            # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
            if any(key.endswith(A__					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
              tensors[key] += temp[key]
            else:
              # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
              lowerCAmelCase_			:      Optional[Any]			=   1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN					) else 0
              # We concatenate these weights accross TP ranks
              lowerCAmelCase_			:      Optional[Any]			=   torch.cat([tensors[key], temp[key]]							,    dim=A__					)

            # Divide by the number of TP the weights we want to average
      for key in tensors.keys():
        if any(key.endswith(A__					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
          lowerCAmelCase_			:      Union[str, Any]			=   tensors[key] / pretraining_tp

      lowerCAmelCase_			:      Optional[int]			=   model.load_state_dict(A__							,    strict=A__					)
      assert not other_keys.unexpected_keys, f'The keys {other_keys.unexpected_keys} are unexpected'
      if missing_keys is None:
        lowerCAmelCase_			:      Any			=   set(other_keys.missing_keys					)
      else:
        lowerCAmelCase_			:      List[Any]			=   missing_keys.intersection(set(other_keys.missing_keys					)					)

    assert not missing_keys, f'The keys {missing_keys} are missing'

    # Save pytorch-model
    os.makedirs(A__							,    exist_ok=A__					)
    lowerCAmelCase_			:      Dict			=   pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
    lowerCAmelCase_			:      Tuple			=   pytorch_dump_folder_path + """/""" + CONFIG_NAME
    print(f'Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'					)
    if config.torch_dtype is not None:
      lowerCAmelCase_			:      Any			=   model.to(config.torch_dtype					)
    torch.save(model.state_dict()							,    A__					)
    print(f'Save configuration file to {pytorch_config_dump_path}'					)
    with open(A__							,    """w"""							,    encoding="""utf-8"""					) as f:
      f.write(config.to_json_string()					)


if __name__ == "__main__":
  __A					:  Tuple											=      argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      "--bloom_checkpoint_path",
      default=None,
      type=str,
      required=True,
      help="Path to the Megatron-LM checkpoint path.",
  )
  parser.add_argument(
      "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
  )
  parser.add_argument(
      "--bloom_config_file",
      default="",
      type=str,
      help=(
          "An optional config json file corresponding to the pre-trained model. \n"
          "This specifies the model architecture."
      ),
  )
  parser.add_argument(
      "--shard_model",
      action="store_true",
      help="An optional setting to shard the output model \nThis enables sharding the converted checkpoint",
  )
  parser.add_argument(
      "--pretraining_tp",
      default=4,
      type=int,
      help="Pretraining TP rank that has been used when training the model in Megatron-LM \n",
  )
  __A					:  Dict											=      parser.parse_args()
  convert_bloom_checkpoint_to_pytorch(
      args.bloom_checkpoint_path,
      args.bloom_config_file,
      args.pytorch_dump_folder_path,
      args.shard_model,
      args.pretraining_tp,
  )



