




'''simple docstring'''

import re
from typing import Callable, List, Optional, Union

import tensorflow as tf


try:
							from tensorflow.keras.optimizers.legacy import Adam
except ImportError:
							from tensorflow.keras.optimizers import Adam






class 						a				(   tf.keras.optimizers.schedules.LearningRateSchedule     ):


				def __init__(		self    :					Optional[Any]       ,				lowercase_    :					float       ,				lowercase_    :					Callable       ,				lowercase_    :					int       ,				lowercase_    :					float = 1.0       ,				lowercase_    :					str = None       ,				):
									super().__init__()
									snake_case_									=				initial_learning_rate
									snake_case_									=				warmup_steps
									snake_case_									=				power
									snake_case_									=				decay_schedule_fn
									snake_case_									=				name


				def __call__(		self    :					Tuple       ,				lowercase_    :					str					):
									with tf.name_scope(self.name or '''WarmUp'''					) as name:
														# Implements polynomial warmup. i.e., if global_step < warmup_steps, the
														# learning rate will be `global_step/num_warmup_steps * init_lr`.
														snake_case_									=				tf.cast(lowercase_       ,				tf.floataa					)
														snake_case_									=				tf.cast(self.warmup_steps       ,				tf.floataa					)
														snake_case_									=				global_step_float / warmup_steps_float
														snake_case_									=				self.initial_learning_rate * tf.math.pow(lowercase_       ,				self.power					)
														return tf.cond(
														    global_step_float < warmup_steps_float       ,				lambda: warmup_learning_rate       ,				lambda: self.decay_schedule_fn(step - self.warmup_steps					)       ,				name=lowercase_       ,				)


				def   A_ (		self    :					Any					):
									return {
									    "initial_learning_rate": self.initial_learning_rate,
									    "decay_schedule_fn": self.decay_schedule_fn,
									    "warmup_steps": self.warmup_steps,
									    "power": self.power,
									    "name": self.name,
									}







def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase = 0.0,      __UpperCAmelCase = 0.9,      __UpperCAmelCase = 0.9_9_9,      __UpperCAmelCase = 1e-8,      __UpperCAmelCase = None,      __UpperCAmelCase = None,      __UpperCAmelCase = 0.0,      __UpperCAmelCase = 1.0,      __UpperCAmelCase = None,      )		->					List[str]:

					'''simple docstring'''

					snake_case_									=				tf.keras.optimizers.schedules.PolynomialDecay(
					    initial_learning_rate=__UpperCAmelCase,      decay_steps=num_train_steps - num_warmup_steps,      end_learning_rate=init_lr * min_lr_ratio,      power=__UpperCAmelCase,      )
					if num_warmup_steps:
										snake_case_									=				WarmUp(
										    initial_learning_rate=__UpperCAmelCase,      decay_schedule_fn=__UpperCAmelCase,      warmup_steps=__UpperCAmelCase,      )
					if weight_decay_rate > 0.0:
										snake_case_									=				AdamWeightDecay(
										    learning_rate=__UpperCAmelCase,      weight_decay_rate=__UpperCAmelCase,      beta_a=__UpperCAmelCase,      beta_a=__UpperCAmelCase,      epsilon=__UpperCAmelCase,      clipnorm=__UpperCAmelCase,      global_clipnorm=__UpperCAmelCase,      exclude_from_weight_decay=['''LayerNorm''', '''layer_norm''', '''bias'''],      include_in_weight_decay=__UpperCAmelCase,      )
					else:
										snake_case_									=				tf.keras.optimizers.Adam(
										    learning_rate=__UpperCAmelCase,      beta_a=__UpperCAmelCase,      beta_a=__UpperCAmelCase,      epsilon=__UpperCAmelCase,      clipnorm=__UpperCAmelCase,      global_clipnorm=__UpperCAmelCase,      )
					# We return the optimizer and the LR scheduler in order to better track the
					# evolution of the LR independently of the optimizer.
					return optimizer, lr_schedule






class 						a				(   _lowerCamelCase     ):


				def __init__(		self    :					Dict       ,				lowercase_    :					Union[float, tf.keras.optimizers.schedules.LearningRateSchedule] = 0.001       ,				lowercase_    :					float = 0.9       ,				lowercase_    :					float = 0.999       ,				lowercase_    :					float = 1e-7       ,				lowercase_    :					bool = False       ,				lowercase_    :					float = 0.0       ,				lowercase_    :					Optional[List[str]] = None       ,				lowercase_    :					Optional[List[str]] = None       ,				lowercase_    :					str = "AdamWeightDecay"       ,				**lowercase_    :					Optional[int]       ,				):
									super().__init__(lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				**lowercase_					)
									snake_case_									=				weight_decay_rate
									snake_case_									=				include_in_weight_decay
									snake_case_									=				exclude_from_weight_decay


				@classmethod
				def   A_ (		cls    :					Dict       ,				lowercase_    :					Union[str, Any]					):
									snake_case_									=				{'''WarmUp''': WarmUp}
									return super(lowercase_       ,				cls					).from_config(lowercase_       ,				custom_objects=lowercase_					)


				def   A_ (		self    :					str       ,				lowercase_    :					Optional[int]       ,				lowercase_    :					List[str]       ,				lowercase_    :					Optional[int]					):
									super(lowercase_       ,				self					)._prepare_local(lowercase_       ,				lowercase_       ,				lowercase_					)
									snake_case_									=				tf.constant(
									    self.weight_decay_rate       ,				name='''adam_weight_decay_rate'''					)


				def   A_ (		self    :					str       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Dict       ,				lowercase_    :					Any					):
									snake_case_									=				self._do_use_weight_decay(var.name					)
									if do_decay:
														return var.assign_sub(
														    learning_rate * var * apply_state[(var.device, var.dtype.base_dtype)]['''weight_decay_rate''']       ,				use_locking=self._use_locking       ,				)
									return tf.no_op()


				def   A_ (		self    :					Union[str, Any]       ,				lowercase_    :					int       ,				lowercase_    :					str=None       ,				**lowercase_    :					List[str]					):
									snake_case_			,snake_case_									=				list(zip(*lowercase_					)					)
									return super(lowercase_       ,				self					).apply_gradients(zip(lowercase_       ,				lowercase_					)       ,				name=lowercase_       ,				**lowercase_					)


				def   A_ (		self    :					List[Any]       ,				lowercase_    :					str       ,				lowercase_    :					str       ,				lowercase_    :					Any					):
									if apply_state is None:
														return self._decayed_lr_t[var_dtype], {}

									snake_case_									=				apply_state or {}
									snake_case_									=				apply_state.get((var_device, var_dtype)					)
									if coefficients is None:
														snake_case_									=				self._fallback_apply_state(lowercase_       ,				lowercase_					)
														snake_case_									=				coefficients

									return coefficients["lr_t"], {"apply_state": apply_state}


				def   A_ (		self    :					Union[str, Any]       ,				lowercase_    :					int       ,				lowercase_    :					Union[str, Any]       ,				lowercase_    :					Optional[int]=None					):
									snake_case_			,snake_case_									=				self._get_lr(var.device       ,				var.dtype.base_dtype       ,				lowercase_					)
									snake_case_									=				self._decay_weights_op(lowercase_       ,				lowercase_       ,				lowercase_					)
									with tf.control_dependencies([decay]					):
														return super(lowercase_       ,				self					)._resource_apply_dense(lowercase_       ,				lowercase_       ,				**lowercase_					)


				def   A_ (		self    :					Optional[Any]       ,				lowercase_    :					Optional[Any]       ,				lowercase_    :					int       ,				lowercase_    :					str       ,				lowercase_    :					List[Any]=None					):
									snake_case_			,snake_case_									=				self._get_lr(var.device       ,				var.dtype.base_dtype       ,				lowercase_					)
									snake_case_									=				self._decay_weights_op(lowercase_       ,				lowercase_       ,				lowercase_					)
									with tf.control_dependencies([decay]					):
														return super(lowercase_       ,				self					)._resource_apply_sparse(lowercase_       ,				lowercase_       ,				lowercase_       ,				**lowercase_					)


				def   A_ (		self    :					Union[str, Any]					):
									snake_case_									=				super().get_config()
									config.update({'''weight_decay_rate''': self.weight_decay_rate}					)
									return config


				def   A_ (		self    :					Optional[int]       ,				lowercase_    :					int					):
									if self.weight_decay_rate == 0:
														return False

									if self._include_in_weight_decay:
														for r in self._include_in_weight_decay:
																			if re.search(lowercase_       ,				lowercase_					) is not None:
																								return True

									if self._exclude_from_weight_decay:
														for r in self._exclude_from_weight_decay:
																			if re.search(lowercase_       ,				lowercase_					) is not None:
																								return False
									return True







class 						a				(   _lowerCamelCase     ):


				def __init__(		self    :					List[Any]					):
									snake_case_									=				[]
									snake_case_									=				None


				@property
				def   A_ (		self    :					Union[str, Any]					):
									if self._accum_steps is None:
														snake_case_									=				tf.Variable(
														    tf.constant(0       ,				dtype=tf.intaa					)       ,				trainable=lowercase_       ,				synchronization=tf.VariableSynchronization.ON_READ       ,				aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA       ,				)

									return self._accum_steps.value()


				@property
				def   A_ (		self    :					Dict					):
									if not self._gradients:
														raise ValueError('''The accumulator should be called first to initialize the gradients'''					)
									return [gradient.value() if gradient is not None else gradient for gradient in self._gradients]


				def __call__(		self    :					Any       ,				lowercase_    :					int					):
									if not self._gradients:
														snake_case_									=				self.step  # Create the step variable.
														self._gradients.extend(
														    [
														        tf.Variable(
														            tf.zeros_like(lowercase_					)       ,				trainable=lowercase_       ,				synchronization=tf.VariableSynchronization.ON_READ       ,				aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA       ,				)
														        if gradient is not None
														        else gradient
														        for gradient in gradients
														    ]					)
									if len(lowercase_					) != len(self._gradients					):
														raise ValueError(F"Expected {len(self._gradients					)} gradients, but got {len(lowercase_					)}"					)

									for accum_gradient, gradient in zip(self._gradients       ,				lowercase_					):
														if accum_gradient is not None and gradient is not None:
																			accum_gradient.assign_add(lowercase_					)

									self._accum_steps.assign_add(1					)


				def   A_ (		self    :					Optional[int]					):
									if not self._gradients:
														return
									self._accum_steps.assign(0					)
									for gradient in self._gradients:
														if gradient is not None:
																			gradient.assign(tf.zeros_like(lowercase_					)					)


import argparse

import torch

from transformers import BertConfig, BertForPreTraining, load_tf_weights_in_bert
from transformers.utils import logging


logging.set_verbosity_info()

def 			lowercase_				(			_lowerCamelCase							:       str		,	_lowerCamelCase							:       List[Any]		,	_lowerCamelCase							:       Dict):
    # Initialise PyTorch model
    lowercase__ :   List[str]    					=				BertConfig.from_json_file(_lowerCamelCase)
    print(f'''Building PyTorch model from configuration: {config}''')
    lowercase__ :   Optional[Any]    					=				BertForPreTraining(_lowerCamelCase)

    # Load weights from tf checkpoint
    load_tf_weights_in_bert(_lowerCamelCase		,	_lowerCamelCase		,	_lowerCamelCase)

    # Save pytorch-model
    print(f'''Save PyTorch model to {pytorch_dump_path}''')
    torch.save(model.state_dict()		,	_lowerCamelCase)


if __name__ == "__main__":
      UpperCamelCase  							=						argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          '''--tf_checkpoint_path''', default=None, type=str, required=True, help='''Path to the TensorFlow checkpoint path.'''
      )
      parser.add_argument(
          '''--bert_config_file''',
          default=None,
          type=str,
          required=True,
          help=(
              '''The config json file corresponding to the pre-trained BERT model. \n'''
              '''This specifies the model architecture.'''
          ),
      )
      parser.add_argument(
          '''--pytorch_dump_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
      )
      UpperCamelCase  							=						parser.parse_args()
      convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)



