





import argparse
import os

import numpy as np
import tensorflow as tf
import torch

from transformers import BertModel





def      _UpperCamelCase  (a__ :BertModel ,      a__ :str ,      a__ :str							):






							"""simple docstring"""





							UpperCamelCase__					  =      ("""dense.weight""", """attention.self.query""", """attention.self.key""", """attention.self.value""")

							UpperCamelCase__					  =      (
							    ("""layer.""", """layer_"""),
							    ("""word_embeddings.weight""", """word_embeddings"""),
							    ("""position_embeddings.weight""", """position_embeddings"""),
							    ("""token_type_embeddings.weight""", """token_type_embeddings"""),
							    (""".""", """/"""),
							    ("""LayerNorm/weight""", """LayerNorm/gamma"""),
							    ("""LayerNorm/bias""", """LayerNorm/beta"""),
							    ("""weight""", """kernel"""),
							)

							if not os.path.isdir(a__							):
														os.makedirs(a__							)

							UpperCamelCase__					  =      model.state_dict()

							def to_tf_var_name(a__ :str							):
														for patt, repl in iter(a__							):
																					UpperCamelCase__					  =      name.replace(a__ ,      a__							)
														return f"""bert/{name}"""

							def create_tf_var(a__ :np.ndarray ,      a__ :str ,      a__ :tf.Session							):
														UpperCamelCase__					  =      tf.dtypes.as_dtype(tensor.dtype							)
														UpperCamelCase__					  =      tf.get_variable(dtype=a__ ,      shape=tensor.shape ,      name=a__ ,      initializer=tf.zeros_initializer()							)
														session.run(tf.variables_initializer([tf_var]							)							)
														session.run(a__							)
														return tf_var

							tf.reset_default_graph()
							with tf.Session() as session:
														for var_name in state_dict:
																					UpperCamelCase__					  =      to_tf_var_name(a__							)
																					UpperCamelCase__					  =      state_dict[var_name].numpy()
																					if any(x in var_name for x in tensors_to_transpose							):
																												UpperCamelCase__					  =      torch_tensor.T
																					UpperCamelCase__					  =      create_tf_var(tensor=a__ ,      name=a__ ,      session=a__							)
																					tf.keras.backend.set_value(a__ ,      a__							)
																					UpperCamelCase__					  =      session.run(a__							)
																					print(f"""Successfully created {tf_name}: {np.allclose(a__ ,      a__							)}"""							)

														UpperCamelCase__					  =      tf.train.Saver(tf.trainable_variables()							)
														saver.save(a__ ,      os.path.join(a__ ,      model_name.replace("""-""" ,      """_"""							) + """.ckpt"""							)							)





def      _UpperCamelCase  (a__ :Dict=None							):






							"""simple docstring"""





							UpperCamelCase__					  =      argparse.ArgumentParser()
							parser.add_argument("""--model_name""" ,      type=a__ ,      required=a__ ,      help="""model name e.g. bert-base-uncased"""							)
							parser.add_argument(
							    """--cache_dir""" ,      type=a__ ,      default=a__ ,      required=a__ ,      help="""Directory containing pytorch model"""							)
							parser.add_argument("""--pytorch_model_path""" ,      type=a__ ,      required=a__ ,      help="""/path/to/<pytorch-model-name>.bin"""							)
							parser.add_argument("""--tf_cache_dir""" ,      type=a__ ,      required=a__ ,      help="""Directory in which to save tensorflow model"""							)
							UpperCamelCase__					  =      parser.parse_args(a__							)

							UpperCamelCase__					  =      BertModel.from_pretrained(
							    pretrained_model_name_or_path=args.model_name ,      state_dict=torch.load(args.pytorch_model_path							) ,      cache_dir=args.cache_dir ,      )

							convert_pytorch_checkpoint_to_tf(model=a__ ,      ckpt_dir=args.tf_cache_dir ,      model_name=args.model_name							)


if __name__ == "__main__":
					main()











from dataclasses import dataclass
from typing import Optional, Tuple

import torch
from torch import nn
from transformers import RobertaPreTrainedModel, XLMRobertaConfig, XLMRobertaModel
from transformers.utils import ModelOutput

@dataclass
class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   Optional[torch.FloatTensor]      =							None
							snake_case  :   torch.FloatTensor      =							None
							snake_case  :   Optional[Tuple[torch.FloatTensor]]      =							None
							snake_case  :   Optional[Tuple[torch.FloatTensor]]      =							None






class 							__SCREAMING_SNAKE_CASE       (  _a							):






							def __init__(    self	,			__lowerCAmelCase=1	,			__lowerCAmelCase=0	,			__lowerCAmelCase=2	,			__lowerCAmelCase=512	,			__lowerCAmelCase="cls"	,			__lowerCAmelCase=False	,			__lowerCAmelCase=True	,			**__lowerCAmelCase	,			):
														super().__init__(pad_token_id=__lowerCAmelCase	,			bos_token_id=__lowerCAmelCase	,			eos_token_id=__lowerCAmelCase	,			**__lowerCAmelCase	)
														UpperCamelCase__					  =      project_dim
														UpperCamelCase__					  =      pooler_fn
														UpperCamelCase__					  =      learn_encoder
														UpperCamelCase__					  =      use_attention_mask






class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   int      =							[r"""pooler""", r"""logit_scale"""]
							snake_case  :   Tuple      =							[r"""position_ids""", r"""predictions.decoder.bias"""]
							snake_case  :   str      =							"""roberta"""
							snake_case  :   Dict      =							RobertaSeriesConfig






							def __init__(    self	,			__lowerCAmelCase	):
														super().__init__(__lowerCAmelCase	)
														UpperCamelCase__					  =      XLMRobertaModel(__lowerCAmelCase	)
														UpperCamelCase__					  =      nn.Linear(config.hidden_size	,			config.project_dim	)
														UpperCamelCase__					  =      getattr(__lowerCAmelCase	,			"""has_pre_transformation"""	,			__lowerCAmelCase	)
														if self.has_pre_transformation:
																					UpperCamelCase__					  =      nn.Linear(config.hidden_size	,			config.project_dim	)
																					UpperCamelCase__					  =      nn.LayerNorm(config.hidden_size	,			eps=config.layer_norm_eps	)
														self.post_init()







							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			):
														UpperCamelCase__					  =      return_dict if return_dict is not None else self.config.use_return_dict

														UpperCamelCase__					  =      self.base_model(
														    input_ids=__lowerCAmelCase	,			attention_mask=__lowerCAmelCase	,			token_type_ids=__lowerCAmelCase	,			position_ids=__lowerCAmelCase	,			head_mask=__lowerCAmelCase	,			inputs_embeds=__lowerCAmelCase	,			encoder_hidden_states=__lowerCAmelCase	,			encoder_attention_mask=__lowerCAmelCase	,			output_attentions=__lowerCAmelCase	,			output_hidden_states=True if self.has_pre_transformation else output_hidden_states	,			return_dict=__lowerCAmelCase	,			)

														if self.has_pre_transformation:
																					UpperCamelCase__					  =      outputs["""hidden_states"""][-2]
																					UpperCamelCase__					  =      self.pre_LN(__lowerCAmelCase	)
																					UpperCamelCase__					  =      self.transformation_pre(__lowerCAmelCase	)

																					return TransformationModelOutput(
																					    projection_state=__lowerCAmelCase	,			last_hidden_state=outputs.last_hidden_state	,			hidden_states=outputs.hidden_states	,			attentions=outputs.attentions	,			)
														else:
																					UpperCamelCase__					  =      self.transformation(outputs.last_hidden_state	)
																					return TransformationModelOutput(
																					    projection_state=__lowerCAmelCase	,			last_hidden_state=outputs.last_hidden_state	,			hidden_states=outputs.hidden_states	,			attentions=outputs.attentions	,			)




