

"""simple docstring"""





from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel

def   __SCREAMING_SNAKE_CASE  (      A_						,     A_						,     A_=None						,     A_=None ):
	if attention_mask is None:
		lowerCAmelCase__	:						List[str]  				=	tf.cast(tf.math.not_equal(_lowerCamelCase						,     config.pad_token_id )						,     tf.inta )
	return {"input_ids": input_ids, "attention_mask": attention_mask}







@require_tf
class 				SCREAMING_SNAKE_CASE					:






					"""simple docstring"""



					lowercase__				 =       OPTConfig
					lowercase__				 =       {}
					lowercase__				 =       "gelu"







					def __init__(       self			:				Any   ,lowercase_			:				Optional[int]   ,lowercase_			:				Optional[Any]=1_3   ,lowercase_			:				List[str]=7   ,lowercase_			:				Tuple=True   ,lowercase_			:				Dict=False   ,lowercase_			:				Dict=9_9   ,lowercase_			:				int=1_6   ,lowercase_			:				Any=2   ,lowercase_			:				int=4   ,lowercase_			:				Optional[int]=4   ,lowercase_			:				Optional[Any]="gelu"   ,lowercase_			:				List[str]=0.1   ,lowercase_			:				Tuple=0.1   ,lowercase_			:				Tuple=2_0   ,lowercase_			:				List[str]=2   ,lowercase_			:				Union[str, Any]=1   ,lowercase_			:				Optional[Any]=0   ,lowercase_			:				Union[str, Any]=1_6   ,lowercase_			:				Dict=1_6   ,):
						lowerCAmelCase__	:						Optional[int]  				=	parent
						lowerCAmelCase__	:						Optional[Any]  				=	batch_size
						lowerCAmelCase__	:						Dict  				=	seq_length
						lowerCAmelCase__	:						Optional[int]  				=	is_training
						lowerCAmelCase__	:						Union[str, Any]  				=	use_labels
						lowerCAmelCase__	:						Tuple  				=	vocab_size
						lowerCAmelCase__	:						Dict  				=	hidden_size
						lowerCAmelCase__	:						Dict  				=	num_hidden_layers
						lowerCAmelCase__	:						Union[str, Any]  				=	num_attention_heads
						lowerCAmelCase__	:						List[str]  				=	intermediate_size
						lowerCAmelCase__	:						List[str]  				=	hidden_act
						lowerCAmelCase__	:						List[str]  				=	hidden_dropout_prob
						lowerCAmelCase__	:						int  				=	attention_probs_dropout_prob
						lowerCAmelCase__	:						Dict  				=	max_position_embeddings
						lowerCAmelCase__	:						Optional[int]  				=	eos_token_id
						lowerCAmelCase__	:						Optional[int]  				=	pad_token_id
						lowerCAmelCase__	:						Dict  				=	bos_token_id
						lowerCAmelCase__	:						List[str]  				=	embed_dim
						lowerCAmelCase__	:						List[str]  				=	word_embed_proj_dim
						lowerCAmelCase__	:						List[str]  				=	False







					def        __lowerCAmelCase (       self			:				str		):
						lowerCAmelCase__	:						Union[str, Any]  				=	ids_tensor([self.batch_size, self.seq_length - 1]   ,self.vocab_size		)
						lowerCAmelCase__	:						List[Any]  				=	tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size		)   ,1		)
						lowerCAmelCase__	:						int  				=	tf.concat([input_ids, eos_tensor]   ,axis=1		)

						lowerCAmelCase__	:						Any  				=	self.config_cls(
						    vocab_size=self.vocab_size   ,hidden_size=self.hidden_size   ,num_hidden_layers=self.num_hidden_layers   ,num_attention_heads=self.num_attention_heads   ,ffn_dim=self.intermediate_size   ,dropout=self.hidden_dropout_prob   ,attention_dropout=self.attention_probs_dropout_prob   ,max_position_embeddings=self.max_position_embeddings   ,eos_token_id=self.eos_token_id   ,bos_token_id=self.bos_token_id   ,pad_token_id=self.pad_token_id   ,embed_dim=self.embed_dim   ,word_embed_proj_dim=self.word_embed_proj_dim   ,is_encoder_decoder=lowercase_   ,**self.config_updates   ,)
						lowerCAmelCase__	:						Union[str, Any]  				=	prepare_opt_inputs_dict(lowercase_   ,lowercase_		)
						return config, inputs_dict







					def        __lowerCAmelCase (       self			:				Union[str, Any]   ,lowercase_			:				int   ,lowercase_			:				Optional[Any]		):
						lowerCAmelCase__	:						Tuple  				=	TFOPTModel(config=lowercase_		)
						lowerCAmelCase__	:						Optional[Any]  				=	inputs_dict["input_ids"]

						lowerCAmelCase__	:						Optional[int]  				=	input_ids[:1, :]
						lowerCAmelCase__	:						int  				=	inputs_dict["attention_mask"][:1, :]
						lowerCAmelCase__	:						List[Any]  				=	1

						# first forward pass
						lowerCAmelCase__	:						Any  				=	model(lowercase_   ,attention_mask=lowercase_   ,use_cache=lowercase_		)

						lowerCAmelCase__	:						Tuple  				=	outputs.to_tuple()

						# create hypothetical next token and extent to next_input_ids
						lowerCAmelCase__	:						Dict  				=	ids_tensor((self.batch_size, 3)   ,config.vocab_size		)
						lowerCAmelCase__	:						str  				=	tf.cast(ids_tensor((self.batch_size, 3)   ,2		)   ,tf.inta		)

						# append to next input_ids and
						lowerCAmelCase__	:						Tuple  				=	tf.concat([input_ids, next_tokens]   ,axis=-1		)
						lowerCAmelCase__	:						Dict  				=	tf.concat([attention_mask, next_attn_mask]   ,axis=-1		)

						lowerCAmelCase__	:						Union[str, Any]  				=	model(lowercase_   ,attention_mask=lowercase_		)[0]
						lowerCAmelCase__	:						Optional[Any]  				=	model(lowercase_   ,attention_mask=lowercase_   ,past_key_values=lowercase_		)[0]

						self.parent.assertEqual(next_tokens.shape[1]   ,output_from_past.shape[1]		)

						# select random slice
						lowerCAmelCase__	:						Any  				=	int(ids_tensor((1,)   ,output_from_past.shape[-1]		)		)
						lowerCAmelCase__	:						Dict  				=	output_from_no_past[:, -3:, random_slice_idx]
						lowerCAmelCase__	:						Any  				=	output_from_past[:, :, random_slice_idx]

						# test that outputs are equal for slice
						tf.debugging.assert_near(lowercase_   ,lowercase_   ,rtol=1E-3		)







@require_tf
class 				SCREAMING_SNAKE_CASE					(		__A    ,							__A    ,							unittest.TestCase       ):






					"""simple docstring"""



					lowercase__				 =       (TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
					lowercase__				 =       (TFOPTForCausalLM,) if is_tf_available() else ()
					lowercase__				 =       (
					    {"feature-extraction": TFOPTModel, "text-generation": TFOPTForCausalLM} if is_tf_available() else {}
					)
					lowercase__				 =       False
					lowercase__				 =       False
					lowercase__				 =       False
					lowercase__				 =       10







					def        __lowerCAmelCase (       self			:				Dict		):
						lowerCAmelCase__	:						List[str]  				=	TFOPTModelTester(self		)
						lowerCAmelCase__	:						Tuple  				=	ConfigTester(self   ,config_class=lowercase_		)







					def        __lowerCAmelCase (       self			:				Tuple		):
						self.config_tester.run_common_tests()







					def        __lowerCAmelCase (       self			:				Union[str, Any]		):
						lowerCAmelCase__	:						str  				=	self.model_tester.prepare_config_and_inputs_for_common()
						self.model_tester.check_decoder_model_past_large_inputs(*lowercase_		)







					def        __lowerCAmelCase (       self			:				Optional[Any]		):
						lowerCAmelCase__	:						List[str]  				=	self.model_tester.prepare_config_and_inputs_for_common()

						def _get_word_embedding_weight(lowercase_			:				Tuple   ,lowercase_			:				str		):
							if hasattr(lowercase_   ,'''weight'''		):
								return embedding_layer.weight
							else:
								# Here we build the word embeddings weights if not exists.
								# And then we retry to get the attribute once built.
								model.build()
								if hasattr(lowercase_   ,'''weight'''		):
									return embedding_layer.weight
								else:
									return None

						for model_class in self.all_model_classes:
							for size in [config.vocab_size - 1_0, config.vocab_size + 1_0]:
								# build the embeddings
								lowerCAmelCase__	:						List[str]  				=	model_class(config=lowercase_		)
								lowerCAmelCase__	:						List[Any]  				=	_get_word_embedding_weight(lowercase_   ,model.get_input_embeddings()		)
								lowerCAmelCase__	:						List[Any]  				=	_get_word_embedding_weight(lowercase_   ,model.get_output_embeddings()		)

								# reshape the embeddings
								model.resize_token_embeddings(lowercase_		)
								lowerCAmelCase__	:						Optional[Any]  				=	_get_word_embedding_weight(lowercase_   ,model.get_input_embeddings()		)
								lowerCAmelCase__	:						Tuple  				=	_get_word_embedding_weight(lowercase_   ,model.get_output_embeddings()		)

								# check that the resized embeddings size matches the desired size.
								lowerCAmelCase__	:						Dict  				=	size if size is not None else config.vocab_size

								self.assertEqual(new_input_embeddings.shape[0]   ,lowercase_		)

								# check that weights remain the same after resizing
								lowerCAmelCase__	:						List[Any]  				=	True
								for pa, pa in zip(old_input_embeddings.value()   ,new_input_embeddings.value()		):
									if tf.math.reduce_sum(tf.math.abs(pa - pa		)		) > 0:
										lowerCAmelCase__	:						List[Any]  				=	False
								self.assertTrue(lowercase_		)

								if old_output_embeddings is not None and new_output_embeddings is not None:
									self.assertEqual(new_output_embeddings.shape[0]   ,lowercase_		)

									lowerCAmelCase__	:						Dict  				=	True
									for pa, pa in zip(old_output_embeddings.value()   ,new_output_embeddings.value()		):
										if tf.math.reduce_sum(tf.math.abs(pa - pa		)		) > 0:
											lowerCAmelCase__	:						List[str]  				=	False
									self.assertTrue(lowercase_		)

def   __SCREAMING_SNAKE_CASE  (      A_ ):
	return tf.constant(_lowerCamelCase						,     dtype=tf.intaa )







@require_tf
class 				SCREAMING_SNAKE_CASE					(		unittest.TestCase       ):






					"""simple docstring"""



					lowercase__				 =       99







					def        __lowerCAmelCase (       self			:				str		):
						lowerCAmelCase__	:						Any  				=	tf.ones((4, 1)   ,dtype=tf.intaa		) * 2
						lowerCAmelCase__	:						Optional[Any]  				=	tf.concat([ids_tensor((4, 6)   ,self.vocab_size - 3		) + 3, eos_column_vector]   ,axis=1		)
						lowerCAmelCase__	:						int  				=	input_ids.shape[0]
						lowerCAmelCase__	:						Any  				=	OPTConfig(
						    vocab_size=self.vocab_size   ,hidden_size=2_4   ,num_hidden_layers=2   ,num_attention_heads=2   ,ffn_dim=3_2   ,max_position_embeddings=4_8   ,eos_token_id=2   ,pad_token_id=1   ,bos_token_id=0   ,)
						return config, input_ids, batch_size







@require_sentencepiece
@require_tf
class 				SCREAMING_SNAKE_CASE					(		unittest.TestCase       ):






					"""simple docstring"""



					@slow
					def        __lowerCAmelCase (       self			:				str		):
						lowerCAmelCase__	:						Dict  				=	TFOPTModel.from_pretrained('''facebook/opt-350m'''		)
						lowerCAmelCase__	:						Dict  				=	_long_tensor([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]		)
						lowerCAmelCase__	:						Tuple  				=	tf.not_equal(lowercase_   ,model.config.pad_token_id		)
						with tf.GradientTape():
							lowerCAmelCase__	:						Union[str, Any]  				=	model(input_ids=lowercase_   ,attention_mask=lowercase_		).last_hidden_state
						lowerCAmelCase__	:						str  				=	(1, 1_1, 5_1_2)
						self.assertEqual(output.shape   ,lowercase_		)
						lowerCAmelCase__	:						str  				=	tf.constant(
						    [[-0.2873, -1.9218, -0.3033], [-1.2710, -0.1338, -0.1902], [0.4095, 0.1214, -1.3121]]		)
						self.assertTrue(np.allclose(output[:, :3, :3]   ,lowercase_   ,atol=4E-3		)		)

						lowerCAmelCase__	:						Union[str, Any]  				=	tf.function(lowercase_   ,jit_compile=lowercase_		)
						lowerCAmelCase__	:						List[str]  				=	xla_generate(lowercase_   ,lowercase_		)[0]
						self.assertTrue(np.allclose(output[:, :3, :3]   ,lowercase_   ,atol=4E-2		)		)







@require_tf
@slow
class 				SCREAMING_SNAKE_CASE					(		unittest.TestCase       ):






					"""simple docstring"""



					def        __lowerCAmelCase (       self			:				Dict		):
						super().setUp()
						lowerCAmelCase__	:						str  				=	"facebook/opt-350m"







					def        __lowerCAmelCase (       self			:				Optional[Any]		):
						lowerCAmelCase__	:						Dict  				=	TFOPTForCausalLM.from_pretrained(self.path_model		)
						lowerCAmelCase__	:						Any  				=	GPTaTokenizer.from_pretrained(self.path_model		)

						lowerCAmelCase__	:						List[str]  				=	[
						    "Today is a beautiful day and I want to",
						    "In the city of",
						    "Paris is the capital of France and",
						    "Computers and mobile phones have taken",
						]
						# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
						lowerCAmelCase__	:						Tuple  				=	tokenizer(lowercase_   ,return_tensors='''tf'''   ,padding=lowercase_   ,add_special_tokens=lowercase_		)
						lowerCAmelCase__	:						List[str]  				=	tf.math.reduce_mean(model(inputs.input_ids   ,attention_mask=inputs.attention_mask		)[0]   ,axis=-1		)
						lowerCAmelCase__	:						Optional[Any]  				=	tf.constant(
						    [
						        [1.3851, -13.8923, -10.5229, -10.7533, -0.2309, -10.2384, -0.5365, -9.0947, -5.1670],
						        [-4.7073, -10.6276, -3.9415, -21.5242, -0.2822, -0.2822, -0.2822, -0.2822, -0.2822],
						        [0.6247, -3.4229, -8.9179, -1.4297, -14.1650, 1.4146, -9.0218, -0.2703, -0.2703],
						        [6.4783, -1.9913, -10.7926, -2.3336, 1.5092, -0.9974, -6.8213, 1.3477, 1.3477],
						    ]		)
						self.assertTrue(np.allclose(lowercase_   ,lowercase_   ,atol=1E-4		)		)

						lowerCAmelCase__	:						Dict  				=	tf.function(lowercase_   ,jit_compile=lowercase_		)
						lowerCAmelCase__	:						Any  				=	tf.math.reduce_mean(xla_generate(inputs.input_ids   ,attention_mask=inputs.attention_mask		)[0]   ,axis=-1		)
						self.assertTrue(np.allclose(lowercase_   ,lowercase_   ,atol=1E-4		)		)







@require_tf
@slow
class 				SCREAMING_SNAKE_CASE					(		unittest.TestCase       ):






					"""simple docstring"""



					@property
					def        __lowerCAmelCase (       self			:				int		):
						return [
						    "Today is a beautiful day and I want",
						    "In the city of",
						    "Paris is the capital of France and",
						    "Computers and mobile phones have taken",
						]







					def        __lowerCAmelCase (       self			:				List[Any]		):
						lowerCAmelCase__	:						List[Any]  				=	"facebook/opt-125m"

						lowerCAmelCase__	:						Optional[Any]  				=	[
						    "Today is a beautiful day and I want to",
						    "In the city of New York, the city",
						    "Paris is the capital of France and the capital",
						    "Computers and mobile phones have taken over the",
						]

						lowerCAmelCase__	:						List[Any]  				=	[]
						lowerCAmelCase__	:						Union[str, Any]  				=	GPTaTokenizer.from_pretrained(lowercase_		)
						lowerCAmelCase__	:						List[Any]  				=	TFOPTForCausalLM.from_pretrained(lowercase_		)

						for prompt in self.prompts:
							lowerCAmelCase__	:						Tuple  				=	tokenizer(lowercase_   ,return_tensors='''tf'''		).input_ids

							lowerCAmelCase__	:						Any  				=	model.generate(lowercase_   ,max_length=1_0		)

							lowerCAmelCase__	:						Optional[int]  				=	tokenizer.batch_decode(lowercase_   ,skip_special_tokens=lowercase_		)
							predicted_outputs += generated_string

						self.assertListEqual(lowercase_   ,lowercase_		)







					def        __lowerCAmelCase (       self			:				List[str]		):
						lowerCAmelCase__	:						Dict  				=	"facebook/opt-350m"

						lowerCAmelCase__	:						Optional[int]  				=	GPTaTokenizer.from_pretrained(lowercase_		)
						lowerCAmelCase__	:						Optional[Any]  				=	TFOPTForCausalLM.from_pretrained(lowercase_		)

						lowerCAmelCase__	:						Tuple  				=	"left"

						# use different length sentences to test batching
						lowerCAmelCase__	:						str  				=	[
						    "Hello, my dog is a little",
						    "Today, I",
						]

						lowerCAmelCase__	:						List[str]  				=	tokenizer(lowercase_   ,return_tensors='''tf'''   ,padding=lowercase_		)
						lowerCAmelCase__	:						Dict  				=	inputs["input_ids"]

						lowerCAmelCase__	:						int  				=	model.generate(input_ids=lowercase_   ,attention_mask=inputs['''attention_mask''']		)

						lowerCAmelCase__	:						Tuple  				=	tokenizer(sentences[0]   ,return_tensors='''tf'''		).input_ids
						lowerCAmelCase__	:						Dict  				=	model.generate(input_ids=lowercase_		)

						lowerCAmelCase__	:						Union[str, Any]  				=	inputs_non_padded.shape[-1] - tf.math.reduce_sum(
						    tf.cast(inputs['''attention_mask'''][-1]   ,tf.intaa		)		)
						lowerCAmelCase__	:						Optional[Any]  				=	tokenizer(sentences[1]   ,return_tensors='''tf'''		).input_ids
						lowerCAmelCase__	:						List[str]  				=	model.generate(input_ids=lowercase_   ,max_length=model.config.max_length - num_paddings		)

						lowerCAmelCase__	:						List[str]  				=	tokenizer.batch_decode(lowercase_   ,skip_special_tokens=lowercase_		)
						lowerCAmelCase__	:						Tuple  				=	tokenizer.decode(output_non_padded[0]   ,skip_special_tokens=lowercase_		)
						lowerCAmelCase__	:						Tuple  				=	tokenizer.decode(output_padded[0]   ,skip_special_tokens=lowercase_		)

						lowerCAmelCase__	:						List[Any]  				=	[
						    "Hello, my dog is a little bit of a dork.\nI'm a little bit",
						    "Today, I was in the middle of a conversation with a friend about the",
						]
						self.assertListEqual(lowercase_   ,lowercase_		)
						self.assertListEqual(lowercase_   ,[non_padded_sentence, padded_sentence]		)







					def        __lowerCAmelCase (       self			:				Tuple		):
						lowerCAmelCase__	:						Optional[int]  				=	"facebook/opt-350m"

						lowerCAmelCase__	:						Union[str, Any]  				=	[
						    "Today is a beautiful day and I want to",
						    "In the city of San Francisco, the city",
						    "Paris is the capital of France and the capital",
						    "Computers and mobile phones have taken over the",
						]

						lowerCAmelCase__	:						Optional[int]  				=	[]
						lowerCAmelCase__	:						Union[str, Any]  				=	GPTaTokenizer.from_pretrained(lowercase_		)
						lowerCAmelCase__	:						Optional[int]  				=	TFOPTForCausalLM.from_pretrained(lowercase_		)

						for prompt in self.prompts:
							lowerCAmelCase__	:						List[str]  				=	tokenizer(lowercase_   ,return_tensors='''tf'''		).input_ids

							lowerCAmelCase__	:						Optional[Any]  				=	model.generate(lowercase_   ,max_length=1_0		)

							lowerCAmelCase__	:						int  				=	tokenizer.batch_decode(lowercase_   ,skip_special_tokens=lowercase_		)
							predicted_outputs += generated_string

						self.assertListEqual(lowercase_   ,lowercase_		)





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCamelCase  							=						logging.get_logger(__name__)

UpperCamelCase  							=						{
    '''YituTech/conv-bert-base''': '''https://huggingface.co/YituTech/conv-bert-base/resolve/main/config.json''',
    '''YituTech/conv-bert-medium-small''': (
        '''https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/config.json'''
    ),
    '''YituTech/conv-bert-small''': '''https://huggingface.co/YituTech/conv-bert-small/resolve/main/config.json''',
    # See all ConvBERT models at https://huggingface.co/models?filter=convbert
}



class   snake_case_       (		__A       ):
   __A	:		List[str]         =				"convbert"






   def __init__(      self				:     Union[str, Any]	,						lowercase_				:     str=3_05_22	,						lowercase_				:     Any=7_68	,						lowercase_				:     Tuple=12	,						lowercase_				:     List[str]=12	,						lowercase_				:     Optional[int]=30_72	,						lowercase_				:     Union[str, Any]="gelu"	,						lowercase_				:     str=0.1	,						lowercase_				:     List[str]=0.1	,						lowercase_				:     Optional[Any]=5_12	,						lowercase_				:     Dict=2	,						lowercase_				:     Union[str, Any]=0.02	,						lowercase_				:     Optional[Any]=1E-12	,						lowercase_				:     Optional[int]=1	,						lowercase_				:     List[Any]=0	,						lowercase_				:     Optional[int]=2	,						lowercase_				:     str=7_68	,						lowercase_				:     Dict=2	,						lowercase_				:     Optional[Any]=9	,						lowercase_				:     Union[str, Any]=1	,						lowercase_				:     Any=None	,						**lowercase_				:     Optional[Any]	,						)					->						Dict:
       super().__init__(
           pad_token_id=lowercase_	,						bos_token_id=lowercase_	,						eos_token_id=lowercase_	,						**lowercase_	,						)

       lowercase__ :   List[str]    					=				vocab_size
       lowercase__ :   Union[str, Any]    					=				hidden_size
       lowercase__ :   Any    					=				num_hidden_layers
       lowercase__ :   List[str]    					=				num_attention_heads
       lowercase__ :   Union[str, Any]    					=				intermediate_size
       lowercase__ :   Optional[Any]    					=				hidden_act
       lowercase__ :   int    					=				hidden_dropout_prob
       lowercase__ :   str    					=				attention_probs_dropout_prob
       lowercase__ :   Union[str, Any]    					=				max_position_embeddings
       lowercase__ :   Optional[int]    					=				type_vocab_size
       lowercase__ :   Tuple    					=				initializer_range
       lowercase__ :   List[str]    					=				layer_norm_eps
       lowercase__ :   List[Any]    					=				embedding_size
       lowercase__ :   Optional[Any]    					=				head_ratio
       lowercase__ :   Dict    					=				conv_kernel_size
       lowercase__ :   Tuple    					=				num_groups
       lowercase__ :   Optional[int]    					=				classifier_dropout



class   snake_case_       (		__A       ):






   @property
   def       __UpperCamelCase     (      self				:     Optional[int]  )					->						Mapping[str, Mapping[int, str]]:
       if self.task == "multiple-choice":
           lowercase__ :   Tuple    					=				{0: "batch", 1: "choice", 2: "sequence"}
       else:
           lowercase__ :   str    					=				{0: "batch", 1: "sequence"}
       return OrderedDict(
           [
               ("input_ids", dynamic_axis),
               ("attention_mask", dynamic_axis),
               ("token_type_ids", dynamic_axis),
           ]  )



