



import math

import tensorflow as tf
from packaging import version


def 		_UpperCAmelCase				(      snake_case			):





					"""simple docstring"""

					_lowerCAmelCase						=    tf.convert_to_tensor(snake_case			)
					_lowerCAmelCase						=    0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0			)	,						x.dtype			)			))

					return x * cdf


def 		_UpperCAmelCase				(      snake_case			):





					"""simple docstring"""

					_lowerCAmelCase						=    tf.convert_to_tensor(snake_case			)
					_lowerCAmelCase						=    tf.cast(math.pi	,						x.dtype			)
					_lowerCAmelCase						=    tf.cast(0.044_715	,						x.dtype			)
					_lowerCAmelCase						=    0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi			) * (x + coeff * tf.pow(snake_case	,						3			))			))

					return x * cdf


def 		_UpperCAmelCase				(      snake_case			):





					"""simple docstring"""

					_lowerCAmelCase						=    tf.convert_to_tensor(snake_case			)

					return x * tf.tanh(tf.math.softplus(snake_case			)			)


def 		_UpperCAmelCase				(      snake_case			):





					"""simple docstring"""

					_lowerCAmelCase						=    tf.convert_to_tensor(snake_case			)
					_lowerCAmelCase						=    tf.cast(0.044_715	,						x.dtype			)
					_lowerCAmelCase						=    tf.cast(0.7_978_845_608	,						x.dtype			)

					return 0.5 * x * (1.0 + tf.tanh(x * coeffa * (1.0 + coeffa * x * x)			))


def 		_UpperCAmelCase				(      snake_case			):





					"""simple docstring"""

					_lowerCAmelCase						=    tf.convert_to_tensor(snake_case			)
					_lowerCAmelCase						=    tf.cast(1.702	,						x.dtype			)
					return x * tf.math.sigmoid(coeff * x			)


def 		_UpperCAmelCase				(      snake_case			):





					"""simple docstring"""

					return tf.clip_by_value(_gelu(snake_case			)	,						-10	,						10			)


def 		_UpperCAmelCase				(      snake_case	,						snake_case=-1			):





					"""simple docstring"""

					_lowerCAmelCase	,	_lowerCAmelCase						=    tf.split(snake_case	,						2	,						axis=snake_case			)
					return a * tf.math.sigmoid(snake_case			)


if version.parse(tf.version.VERSION) >= version.parse("""2.4"""):


	def 		_UpperCAmelCase				(      snake_case			):





						"""simple docstring"""

						return tf.keras.activations.gelu(snake_case	,						approximate=snake_case			)

	A__	      =       tf.keras.activations.gelu
	A__	      =       approximate_gelu_wrap
else:
	A__	      =       _gelu
	A__	      =       _gelu_new


A__	      =       {
    """gelu""": gelu,
    """gelu_10""": gelu_aa,
    """gelu_fast""": gelu_fast,
    """gelu_new""": gelu_new,
    """glu""": glu,
    """mish""": mish,
    """quick_gelu""": quick_gelu,
    """relu""": tf.keras.activations.relu,
    """sigmoid""": tf.keras.activations.sigmoid,
    """silu""": tf.keras.activations.swish,
    """swish""": tf.keras.activations.swish,
    """tanh""": tf.keras.activations.tanh,
}


def 		_UpperCAmelCase				(      snake_case			):





					"""simple docstring"""

					if activation_string in ACTaFN:
										return ACTaFN[activation_string]
					else:
										raise KeyError(F'function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()			)}'			)





"""simple docstring"""



import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
 UpperCAmelCase__          = argparse.ArgumentParser(
     description=(
         'Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned'
         ' Distillation'
     )
 )
 parser.add_argument('--model_type', default='bert', choices=['bert'])
 parser.add_argument('--model_name', default='bert-base-uncased', type=str)
 parser.add_argument('--dump_checkpoint', default='serialization_dir/tf_bert-base-uncased_0247911.pth', type=str)
 parser.add_argument('--vocab_transform', action='store_true')
 UpperCAmelCase__          = parser.parse_args()

 if args.model_type == "bert":
  UpperCAmelCase__          = BertForMaskedLM.from_pretrained(args.model_name)
  UpperCAmelCase__          = 'bert'
 else:
  raise ValueError('args.model_type should be "bert".')

 UpperCAmelCase__          = model.state_dict()
 UpperCAmelCase__          = {}

 for w in ["word_embeddings", "position_embeddings"]:
  UpperCAmelCase__          = state_dict[F"{prefix}.embeddings.{w}.weight"]
 for w in ["weight", "bias"]:
  UpperCAmelCase__          = state_dict[F"{prefix}.embeddings.LayerNorm.{w}"]

 UpperCAmelCase__          = 0
 for teacher_idx in [0, 2, 4, 7, 9, 11]:
  for w in ["weight", "bias"]:
   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}"
   ]
   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}"
   ]
   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}"
   ]

   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}"
   ]
   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}"
   ]

   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}"
   ]
   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}"
   ]
   UpperCAmelCase__          = state_dict[
       F"{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}"
   ]
  std_idx += 1

 UpperCAmelCase__          = state_dict['cls.predictions.decoder.weight']
 UpperCAmelCase__          = state_dict['cls.predictions.bias']
 if args.vocab_transform:
  for w in ["weight", "bias"]:
   UpperCAmelCase__          = state_dict[F"cls.predictions.transform.dense.{w}"]
   UpperCAmelCase__          = state_dict[F"cls.predictions.transform.LayerNorm.{w}"]

 print(F"N layers selected for distillation: {std_idx}")
 print(F"Number of params transferred for distillation: {len(compressed_sd.keys())}")

 print(F"Save transferred checkpoint to {args.dump_checkpoint}.")
 torch.save(compressed_sd, args.dump_checkpoint)



