
"""simple docstring"""



import logging
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.utilities import rank_zero_only
from utils_rag import save_json
def 					_UpperCAmelCase      ( __lowerCamelCase   :       Any		)     ->					Union[str, Any]:
  _snake_case					     =       filter(lambda __lowerCamelCase		: p.requires_grad ,					model.parameters()		)
  _snake_case					     =       sum([np.prod(p.size()		) for p in model_parameters]		)
  return params


UpperCAmelCase__          = logging.getLogger(__name__)
def 					_UpperCAmelCase      ( __lowerCamelCase   :       Union[str, Any] ,					__lowerCamelCase   :       List[str]		)     ->					Union[str, Any]:
  if metric == "rouge2":
    _snake_case					     =       '''{val_avg_rouge2:.4f}-{step_count}'''
  elif metric == "bleu":
    _snake_case					     =       '''{val_avg_bleu:.4f}-{step_count}'''
  elif metric == "em":
    _snake_case					     =       '''{val_avg_em:.4f}-{step_count}'''
  elif metric == "loss":
    _snake_case					     =       '''{val_avg_loss:.4f}-{step_count}'''
  else:
    raise NotImplementedError(
        f'''seq2seq callbacks only support rouge2 and bleu, got {metric}, You can make your own by adding to this'''
        ''' function.'''		)

  _snake_case					     =       ModelCheckpoint(
      dirpath=__lowerCamelCase ,					filename=__lowerCamelCase ,					monitor=f'''val_{metric}''' ,					mode='''max''' ,					save_top_k=1 ,					every_n_epochs=1 ,					)

  return checkpoint_callback
def 					_UpperCAmelCase      ( __lowerCamelCase   :       List[str] ,					__lowerCamelCase   :       Optional[Any]		)     ->					List[Any]:
  return EarlyStopping(
      monitor=f'''val_{metric}''' ,					mode='''min''' if '''loss''' in metric else '''max''' ,					patience=__lowerCamelCase ,					verbose=__lowerCamelCase ,					)

class       lowerCAmelCase__				(						pl.Callback	):







     def      lowercase	(							self			:			Union[str, Any]    ,      _lowerCamelCase			:			Union[str, Any]    ,      _lowerCamelCase			:			List[str]			):
       _snake_case					     =       {f'''lr_group_{i}''': param['''lr'''] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups			)}
       pl_module.logger.log_metrics(_lowerCamelCase			)







     @rank_zero_only
     def      lowercase	(							self			:			Tuple    ,      _lowerCamelCase			:			pl.Trainer    ,      _lowerCamelCase			:			pl.LightningModule    ,      _lowerCamelCase			:			str    ,      _lowerCamelCase			:			str=True			):
       logger.info(f'''***** {type_path} results at step {trainer.global_step:05d} *****'''			)
       _snake_case					     =       trainer.callback_metrics
       trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ['''log''', '''progress_bar''', '''preds''']}			)
       # Log results
       _snake_case					     =       Path(pl_module.hparams.output_dir			)
       if type_path == "test":
         _snake_case					     =       od / '''test_results.txt'''
         _snake_case					     =       od / '''test_generations.txt'''
       else:
         # this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
         # If people want this it will be easy enough to add back.
         _snake_case					     =       od / f'''{type_path}_results/{trainer.global_step:05d}.txt'''
         _snake_case					     =       od / f'''{type_path}_generations/{trainer.global_step:05d}.txt'''
         results_file.parent.mkdir(exist_ok=_lowerCamelCase			)
         generations_file.parent.mkdir(exist_ok=_lowerCamelCase			)
       with open(_lowerCamelCase    ,      '''a+'''			) as writer:
         for key in sorted(_lowerCamelCase			):
           if key in ["log", "progress_bar", "preds"]:
             continue
           _snake_case					     =       metrics[key]
           if isinstance(_lowerCamelCase    ,      torch.Tensor			):
             _snake_case					     =       val.item()
           _snake_case					     =       f'''{key}: {val:.6f}\n'''
           writer.write(_lowerCamelCase			)

       if not save_generations:
         return

       if "preds" in metrics:
         _snake_case					     =       '''\n'''.join(metrics['''preds''']			)
         generations_file.open('''w+'''			).write(_lowerCamelCase			)







     @rank_zero_only
     def      lowercase	(							self			:			str    ,      _lowerCamelCase			:			Union[str, Any]    ,      _lowerCamelCase			:			Dict			):
       try:
         _snake_case					     =       pl_module.model.model.num_parameters()
       except AttributeError:
         _snake_case					     =       pl_module.model.num_parameters()

       _snake_case					     =       count_trainable_parameters(_lowerCamelCase			)
       # mp stands for million parameters
       trainer.logger.log_metrics({'''n_params''': npars, '''mp''': npars / 1e6, '''grad_mp''': n_trainable_pars / 1e6}			)







     @rank_zero_only
     def      lowercase	(							self			:			List[Any]    ,      _lowerCamelCase			:			pl.Trainer    ,      _lowerCamelCase			:			pl.LightningModule			):
       save_json(pl_module.metrics    ,      pl_module.metrics_save_path			)
       return self._write_logs(_lowerCamelCase    ,      _lowerCamelCase    ,      '''test'''			)







     @rank_zero_only
     def      lowercase	(							self			:			int    ,      _lowerCamelCase			:			pl.Trainer    ,      _lowerCamelCase			:			Any			):
       save_json(pl_module.metrics    ,      pl_module.metrics_save_path			)
       # Uncommenting this will save val generations
       # return self._write_logs(trainer, pl_module, "valid")





"""simple docstring"""



import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
 UpperCAmelCase__          = argparse.ArgumentParser(
     description=(
         'Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned'
         ' Distillation'
     )
 )
 parser.add_argument('--model_type', default='bert', choices=['bert'])
 parser.add_argument('--model_name', default='bert-base-uncased', type=str)
 parser.add_argument('--dump_checkpoint', default='serialization_dir/tf_bert-base-uncased_0247911.pth', type=str)
 parser.add_argument('--vocab_transform', action='store_true')
 UpperCAmelCase__          = parser.parse_args()

 if args.model_type == "bert":
  UpperCAmelCase__          = BertForMaskedLM.from_pretrained(args.model_name)
  UpperCAmelCase__          = 'bert'
 else:
  raise ValueError('args.model_type should be "bert".')

 UpperCAmelCase__          = model.state_dict()
 UpperCAmelCase__          = {}

 for w in ["word_embeddings", "position_embeddings"]:
  UpperCAmelCase__          = state_dict[F"{prefix}.embeddings.{w}.weight"]
 for w in ["weight", "bias"]:
  UpperCAmelCase__          = state_dict[F"{prefix}.embeddings.LayerNorm.{w}"]

 UpperCAmelCase__          = 0
 for teacher_idx in [0, 2, 4, 7, 9, 11]:
  for w in ["weight", "bias"]:
   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}"
   ]
   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}"
   ]
   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}"
   ]

   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}"
   ]
   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}"
   ]

   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}"
   ]
   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}"
   ]
   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}"
   ]
  std_idx += 1

 UpperCAmelCase__          = state_dict['cls.predictions.decoder.weight']
 UpperCAmelCase__          = state_dict['cls.predictions.bias']
 if args.vocab_transform:
  for w in ["weight", "bias"]:
   UpperCAmelCase__          = state_dict[F"cls.predictions.transform.dense.{w}"]
   UpperCAmelCase__          = state_dict[F"cls.predictions.transform.LayerNorm.{w}"]

 print(F"N layers selected for distillation: {std_idx}")
 print(F"Number of params transferred for distillation: {len(compressed_sd.keys())}")

 print(F"Save transferred checkpoint to {args.dump_checkpoint}.")
 torch.save(compressed_sd, args.dump_checkpoint)



