




"""simple docstring"""






from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
			from transformers import (
			    TFAutoModel,
			    TFAutoModelForCausalLM,
			    TFAutoModelForMaskedLM,
			    TFAutoModelForPreTraining,
			    TFAutoModelForQuestionAnswering,
			    TFAutoModelForSeqaSeqLM,
			    TFAutoModelForSequenceClassification,
			    TFAutoModelForTableQuestionAnswering,
			    TFAutoModelForTokenClassification,
			    TFAutoModelWithLMHead,
			    TFBertForMaskedLM,
			    TFBertForPreTraining,
			    TFBertForQuestionAnswering,
			    TFBertForSequenceClassification,
			    TFBertModel,
			    TFFunnelBaseModel,
			    TFFunnelModel,
			    TFGPTaLMHeadModel,
			    TFRobertaForMaskedLM,
			    TFTaForConditionalGeneration,
			    TFTapasForQuestionAnswering,
			)
			from transformers.models.auto.modeling_tf_auto import (
			    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
			    TF_MODEL_FOR_MASKED_LM_MAPPING,
			    TF_MODEL_FOR_PRETRAINING_MAPPING,
			    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
			    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
			    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
			    TF_MODEL_MAPPING,
			)
			from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST






class lowerCAmelCase__	(     lowercase						):






			'''simple docstring'''



			lowerCamelCase__      =			"""new-model"""


if is_tf_available():






			class lowerCAmelCase__	(     lowercase						):






						'''simple docstring'''



						lowerCamelCase__      =			NewModelConfig






@require_tf
class lowerCAmelCase__	(     unittest.TestCase						):






			'''simple docstring'''



			@slow
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[str]    =    'bert-base-cased'
							_lowerCamelCase     :			Union[str, Any]    =    AutoConfig.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							_lowerCamelCase     :			Union[str, Any]    =    TFAutoModel.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[str]    =    'bert-base-cased'
							_lowerCamelCase     :			int    =    AutoConfig.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							_lowerCamelCase     :			int    =    TFAutoModelForPreTraining.from_pretrained(lowercase  )
							self.assertIsNotNone(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			Dict    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			int    =    TFAutoModelForCausalLM.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			str    =    TFAutoModelForCausalLM.from_pretrained(lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			List[Any]    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			str    =    TFAutoModelWithLMHead.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			Tuple    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Optional[int]    =    TFAutoModelForMaskedLM.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			Tuple    =    TFAutoModelForMaskedLM.from_pretrained(lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											_lowerCamelCase     :			Optional[int]    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Tuple    =    TFAutoModelForSeqaSeqLM.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			Tuple    =    TFAutoModelForSeqaSeqLM.from_pretrained(lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							for model_name in ["bert-base-uncased"]:
											_lowerCamelCase     :			str    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Union[str, Any]    =    TFAutoModelForSequenceClassification.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			def 					A_    (  self  ):
							# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							for model_name in ["bert-base-uncased"]:
											_lowerCamelCase     :			Optional[Any]    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			List[str]    =    TFAutoModelForQuestionAnswering.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			@slow
			@require_tensorflow_probability
			def 					A_    (  self  ):
							for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
											_lowerCamelCase     :			Dict    =    AutoConfig.from_pretrained(lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )

											_lowerCamelCase     :			Dict    =    TFAutoModelForTableQuestionAnswering.from_pretrained(lowercase  )
											_lowerCamelCase, _lowerCamelCase     :			List[Any]    =    TFAutoModelForTableQuestionAnswering.from_pretrained(
											    lowercase  ,      output_loading_info=lowercase  )
											self.assertIsNotNone(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			def 					A_    (  self  ):
							_lowerCamelCase     :			int    =    TFAutoModelWithLMHead.from_pretrained(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
							self.assertEqual(model.num_parameters()  ,      14410  )
							self.assertEqual(model.num_parameters(only_trainable=lowercase  )  ,      14410  )
			def 					A_    (  self  ):
							_lowerCamelCase     :			Any    =    TFAutoModelWithLMHead.from_pretrained(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )
							self.assertEqual(model.num_parameters()  ,      14410  )
							self.assertEqual(model.num_parameters(only_trainable=lowercase  )  ,      14410  )
			def 					A_    (  self  ):
							# For the auto model mapping, FunnelConfig has two models: FunnelModel and FunnelBaseModel
							_lowerCamelCase     :			List[str]    =    TFAutoModel.from_pretrained('sgugger/funnel-random-tiny'  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							_lowerCamelCase     :			Optional[int]    =    copy.deepcopy(model.config  )
							_lowerCamelCase     :			Dict    =    ['FunnelBaseModel']
							_lowerCamelCase     :			List[Any]    =    TFAutoModel.from_config(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )

							with tempfile.TemporaryDirectory() as tmp_dir:
											model.save_pretrained(lowercase  )
											_lowerCamelCase     :			Tuple    =    TFAutoModel.from_pretrained(lowercase  )
											self.assertIsInstance(lowercase  ,      lowercase  )
			def 					A_    (  self  ):
							try:
											AutoConfig.register('new-model'  ,      lowercase  )

											_lowerCamelCase     :			Tuple    =    [
											    TFAutoModel,
											    TFAutoModelForCausalLM,
											    TFAutoModelForMaskedLM,
											    TFAutoModelForPreTraining,
											    TFAutoModelForQuestionAnswering,
											    TFAutoModelForSequenceClassification,
											    TFAutoModelForTokenClassification,
											]

											for auto_class in auto_classes:
															with self.subTest(auto_class.__name__  ):
																			# Wrong config class will raise an error
																			with self.assertRaises(lowercase  ):
																							auto_class.register(lowercase  ,      lowercase  )
																			auto_class.register(lowercase  ,      lowercase  )
																			# Trying to register something existing in the Transformers library will raise an error
																			with self.assertRaises(lowercase  ):
																							auto_class.register(lowercase  ,      lowercase  )

																			# Now that the config is registered, it can be used as any other config with the auto-API
																			_lowerCamelCase     :			Optional[Any]    =    BertModelTester(self  ).get_config()
																			_lowerCamelCase     :			Dict    =    NewModelConfig(**tiny_config.to_dict()  )
																			_lowerCamelCase     :			int    =    auto_class.from_config(lowercase  )
																			self.assertIsInstance(lowercase  ,      lowercase  )

																			with tempfile.TemporaryDirectory() as tmp_dir:
																							model.save_pretrained(lowercase  )
																							_lowerCamelCase     :			List[Any]    =    auto_class.from_pretrained(lowercase  )
																							self.assertIsInstance(lowercase  ,      lowercase  )

							finally:
											if "new-model" in CONFIG_MAPPING._extra_content:
															del CONFIG_MAPPING._extra_content["new-model"]
											for mapping in (
											    TF_MODEL_MAPPING,
											    TF_MODEL_FOR_PRETRAINING_MAPPING,
											    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
											    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
											    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
											    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
											    TF_MODEL_FOR_MASKED_LM_MAPPING,
											):
															if NewModelConfig in mapping._extra_content:
																			del mapping._extra_content[NewModelConfig]
			def 					A_    (  self  ):
							with self.assertRaisesRegex(
							    lowercase  ,      'bert-base is not a local folder and is not a valid model identifier'  ):
											_lowerCamelCase     :			Union[str, Any]    =    TFAutoModel.from_pretrained('bert-base'  )
			def 					A_    (  self  ):
							with self.assertRaisesRegex(
							    lowercase  ,      r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'  ):
											_lowerCamelCase     :			str    =    TFAutoModel.from_pretrained(lowercase  ,      revision='aaaaaa'  )
			def 					A_    (  self  ):
							with self.assertRaisesRegex(
							    lowercase  ,      'hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'  ,      ):
											_lowerCamelCase     :			Tuple    =    TFAutoModel.from_pretrained('hf-internal-testing/config-no-model'  )
			def 					A_    (  self  ):
							with self.assertRaisesRegex(lowercase  ,      'Use `from_pt=True` to load this model'  ):
											_lowerCamelCase     :			Tuple    =    TFAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only'  )






			def 					A_    (  self  ):
							# Make sure we have cached the model.
							_lowerCamelCase     :			Optional[int]    =    TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'  )
							with RequestCounter() as counter:
											_lowerCamelCase     :			Optional[int]    =    TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'  )
											self.assertEqual(counter.get_request_count  ,      0  )
											self.assertEqual(counter.head_request_count  ,      1  )
											self.assertEqual(counter.other_request_count  ,      0  )

							# With a sharded checkpoint
							_lowerCamelCase     :			int    =    TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'  )
							with RequestCounter() as counter:
											_lowerCamelCase     :			List[Any]    =    TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'  )
											self.assertEqual(counter.get_request_count  ,      0  )
											self.assertEqual(counter.head_request_count  ,      1  )
											self.assertEqual(counter.other_request_count  ,      0  )





"""simple docstring"""






from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


lowercase__     						=						"""\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
"""

lowercase__     						=						"""\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
"""

lowercase__     						=						"""
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for 'record': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'prediction_text': the predicted answer text
        - for 'multirc': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question-answer pair as specified by the dataset
            - 'prediction': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for 'record': list of question-answers dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'answers': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for 'record':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1': F1 score
    - for 'multirc':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1_m': Per-question macro-F1 score
        - 'f1_a': Average F1 score over all answers
    - for 'axb':
        'matthews_correlation': Matthew Correlation
    - for 'cb':
        - 'accuracy': Accuracy
        - 'f1': F1 score
    - for all others:
        - 'accuracy': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')
    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]
    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')
    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""







def      _snake_case			(				lowercase__							,    lowercase__		):
				return float((preds == labels).mean()		)







def      _snake_case			(				lowercase__							,    lowercase__							,    lowercase__="binary"		):
				_lowerCamelCase     :			str    =    simple_accuracy(lowercase__							,    lowercase__		)
				_lowerCamelCase     :			Any    =    float(fa_score(y_true=lowercase__							,    y_pred=lowercase__							,    average=lowercase__		)		)
				return {
				    "accuracy": acc,
				    "f1": fa,
				}







def      _snake_case			(				lowercase__							,    lowercase__		):
				_lowerCamelCase     :			Any    =    {}
				for id_pred, label in zip(lowercase__							,    lowercase__		):
								_lowerCamelCase     :			Tuple    =    f'''{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}'''
								_lowerCamelCase     :			Union[str, Any]    =    id_pred['prediction']
								if question_id in question_map:
												question_map[question_id].append((pred, label)		)
								else:
												_lowerCamelCase     :			Optional[Any]    =    [(pred, label)]
				_lowerCamelCase, _lowerCamelCase     :			Optional[int]    =    [], []
				for question, preds_labels in question_map.items():
								_lowerCamelCase, _lowerCamelCase     :			Tuple    =    zip(*lowercase__		)
								_lowerCamelCase     :			List[str]    =    fa_score(y_true=lowercase__							,    y_pred=lowercase__							,    average='macro'		)
								fas.append(lowercase__		)
								_lowerCamelCase     :			int    =    int(sum(pred == label for pred, label in preds_labels		) == len(lowercase__		)		)
								ems.append(lowercase__		)
				_lowerCamelCase     :			Optional[Any]    =    float(sum(lowercase__		) / len(lowercase__		)		)
				_lowerCamelCase     :			Optional[int]    =    sum(lowercase__		) / len(lowercase__		)
				_lowerCamelCase     :			List[Any]    =    float(fa_score(y_true=lowercase__							,    y_pred=[id_pred['prediction'] for id_pred in ids_preds]		)		)
				return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION,	_KWARGS_DESCRIPTION						)
class lowerCAmelCase__	(     datasets.Metric						):






			'''simple docstring'''



			def 					A_    (  self  ):
							if self.config_name not in [
							    "boolq",
							    "cb",
							    "copa",
							    "multirc",
							    "record",
							    "rte",
							    "wic",
							    "wsc",
							    "wsc.fixed",
							    "axb",
							    "axg",
							]:
											raise KeyError(
											    'You should supply a configuration name selected in '
											    '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'  )
							return datasets.MetricInfo(
							    description=_DESCRIPTION  ,      citation=_CITATION  ,      inputs_description=_KWARGS_DESCRIPTION  ,      features=datasets.Features(self._get_feature_types()  )  ,      codebase_urls=[]  ,      reference_urls=[]  ,      format='numpy' if not self.config_name == 'record' and not self.config_name == 'multirc' else None  ,      )
			def 					A_    (  self  ):
							if self.config_name == "record":
											return {
											    "predictions": {
											        "idx": {
											            "passage": datasets.Value('int64'  ),
											            "query": datasets.Value('int64'  ),
											        },
											        "prediction_text": datasets.Value('string'  ),
											    },
											    "references": {
											        "idx": {
											            "passage": datasets.Value('int64'  ),
											            "query": datasets.Value('int64'  ),
											        },
											        "answers": datasets.Sequence(datasets.Value('string'  )  ),
											    },
											}
							elif self.config_name == "multirc":
											return {
											    "predictions": {
											        "idx": {
											            "answer": datasets.Value('int64'  ),
											            "paragraph": datasets.Value('int64'  ),
											            "question": datasets.Value('int64'  ),
											        },
											        "prediction": datasets.Value('int64'  ),
											    },
											    "references": datasets.Value('int64'  ),
											}
							else:
											return {
											    "predictions": datasets.Value('int64'  ),
											    "references": datasets.Value('int64'  ),
											}






			def 					A_    (  self  ,      lowercase  ,      lowercase  ):
							if self.config_name == "axb":
											return {"matthews_correlation": matthews_corrcoef(lowercase  ,      lowercase  )}
							elif self.config_name == "cb":
											return acc_and_fa(lowercase  ,      lowercase  ,      fa_avg='macro'  )
							elif self.config_name == "record":
											_lowerCamelCase     :			List[str]    =    [
											    {
											        'qas': [
											            {'id': ref['idx']['query'], 'answers': [{'text': ans} for ans in ref['answers']]}
											            for ref in references
											        ]
											    }
											]
											_lowerCamelCase     :			Union[str, Any]    =    {pred['idx']['query']: pred['prediction_text'] for pred in predictions}
											return evaluate_record(lowercase  ,      lowercase  )[0]
							elif self.config_name == "multirc":
											return evaluate_multirc(lowercase  ,      lowercase  )
							elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
											return {"accuracy": simple_accuracy(lowercase  ,      lowercase  )}
							else:
											raise KeyError(
											    'You should supply a configuration name selected in '
											    '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'  )