from scipy.stats import pearsonr

import datasets


A   :     Any       =				"\nPearson correlation coefficient and p-value for testing non-correlation.\nThe Pearson correlation coefficient measures the linear relationship between two datasets. The calculation of the p-value relies on the assumption that each dataset is normally distributed. Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases.\nThe p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets.\n"


A   :     List[str]       =				"\nArgs:\n    predictions (`list` of `int`): Predicted class labels, as returned by a model.\n    references (`list` of `int`): Ground truth labels.\n    return_pvalue (`boolean`): If `True`, returns the p-value, along with the correlation coefficient. If `False`, returns only the correlation coefficient. Defaults to `False`.\n\nReturns:\n    pearsonr (`float`): Pearson correlation coefficient. Minimum possible value is -1. Maximum possible value is 1. Values of 1 and -1 indicate exact linear positive and negative relationships, respectively. A value of 0 implies no correlation.\n    p-value (`float`): P-value, which roughly indicates the probability of an The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate higher probabilities.\n\nExamples:\n\n    Example 1-A simple example using only predictions and references.\n        >>> pearsonr_metric = datasets.load_metric(\"pearsonr\")\n        >>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5])\n        >>> print(round(results['pearsonr'], 2))\n        -0.74\n\n    Example 2-The same as Example 1, but that also returns the `p-value`.\n        >>> pearsonr_metric = datasets.load_metric(\"pearsonr\")\n        >>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5], return_pvalue=True)\n        >>> print(sorted(list(results.keys())))\n        ['p-value', 'pearsonr']\n        >>> print(round(results['pearsonr'], 2))\n        -0.74\n        >>> print(round(results['p-value'], 2))\n        0.15\n"


A   :     Optional[Any]       =				"\n@article{2020SciPy-NMeth,\nauthor  = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and\n      Haberland, Matt and Reddy, Tyler and Cournapeau, David and\n      Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and\n      Bright, Jonathan and {van der Walt}, St{\'e}fan J. and\n      Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and\n      Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and\n      Kern, Robert and Larson, Eric and Carey, C J and\n      Polat, Ilhan and Feng, Yu and Moore, Eric W. and\n      {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and\n      Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and\n      Harris, Charles R. and Archibald, Anne M. and\n      Ribeiro, Antonio H. and Pedregosa, Fabian and\n      {van Mulbregt}, Paul and {SciPy 1.0 Contributors}},\ntitle   = {{{SciPy} 1.0: Fundamental Algorithms for Scientific\n      Computing in Python}},\njournal = {Nature Methods},\nyear    = {2020},\nvolume  = {17},\npages   = {261--272},\nadsurl  = {https://rdcu.be/b08Wh},\ndoi = {10.1038/s41592-019-0686-2},\n}\n"


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,					_KWARGS_DESCRIPTION    )
class      lowerCamelCase    (datasets.Metric    ):
  """simple docstring"""







  def       __A   (							self       :				int      )     ->		List[str]:
        return datasets.MetricInfo(
            description=_DESCRIPTION     ,   citation=_CITATION     ,   inputs_description=_KWARGS_DESCRIPTION     ,   features=datasets.Features(
                {
                    "predictions": datasets.Value("float"      ),
                    "references": datasets.Value("float"      ),
                }      )     ,   reference_urls=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.pearsonr.html"]     ,   )







  def       __A   (							self       :				str     ,   __magic_name__       :				Optional[Any]     ,   __magic_name__       :				Tuple     ,   __magic_name__       :				str=False      )     ->		str:
        if return_pvalue:
              SCREAMING_SNAKE_CASE_      =     pearsonr(__magic_name__     ,   __magic_name__      )
              return {"pearsonr": results[0], "p-value": results[1]}
        else:
              return {"pearsonr": float(pearsonr(__magic_name__     ,   __magic_name__      )[0]      )}



import logging
import os
from dataclasses import dataclass, field
from typing import Dict, Optional

import numpy as np
from utils_multiple_choice import MultipleChoiceDataset, Split, processors

import transformers
from transformers import (
    AutoConfig,
    AutoModelForMultipleChoice,
    AutoTokenizer,
    DataCollatorWithPadding,
    EvalPrediction,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import is_main_process


A   :     Any       =				logging.getLogger(__name__)


def    a__       (						__UpperCamelCase   ,					__UpperCamelCase ):
      return (preds == labels).mean()


@dataclass
class      lowerCamelCase    :
  """simple docstring"""







  lowerCamelCase__   =  field(
      metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}    )
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''Pretrained config name or path if not the same as model_name'''}    )
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''Pretrained tokenizer name or path if not the same as model_name'''}    )
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}     ,					)



@dataclass
class      lowerCamelCase    :
  """simple docstring"""







  lowerCamelCase__   =  field(metadata={'''help''': '''The name of the task to train on: ''' + ''', '''.join(processors.keys()    )}    )
  lowerCamelCase__   =  field(metadata={'''help''': '''Should contain the data files for the task.'''}    )
  lowerCamelCase__   =  field(
      default=1_2_8     ,					metadata={
          '''help''': (
              '''The maximum total input sequence length after tokenization. Sequences longer '''
              '''than this will be truncated, sequences shorter will be padded.'''
          )
      }     ,					)
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''Overwrite the cached training and evaluation sets'''}    )



def    a__       (						):
      # See all possible arguments in src/transformers/training_args.py
      # or by passing the --help flag to this script.
      # We now keep distinct sets of args, for a cleaner separation of concerns.

      SCREAMING_SNAKE_CASE_      =     HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments) )
      SCREAMING_SNAKE_CASE_   ,       SCREAMING_SNAKE_CASE_   ,       SCREAMING_SNAKE_CASE_      =     parser.parse_args_into_dataclasses()

      if (
          os.path.exists(training_args.output_dir )
          and os.listdir(training_args.output_dir )
          and training_args.do_train
          and not training_args.overwrite_output_dir
      ):
            raise ValueError(
                F'''Output directory ({training_args.output_dir}) already exists and is not empty. Use'''
                " --overwrite_output_dir to overcome." )

      # Setup logging
      logging.basicConfig(
          format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"   ,					datefmt="%m/%d/%Y %H:%M:%S"   ,					level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN   ,					)
      logger.warning(
          "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"   ,					training_args.local_rank   ,					training_args.device   ,					training_args.n_gpu   ,					bool(training_args.local_rank != -1 )   ,					training_args.fpaa   ,					)
      # Set the verbosity to info of the Transformers logger (on main process only):
      if is_main_process(training_args.local_rank ):
            transformers.utils.logging.set_verbosity_info()
            transformers.utils.logging.enable_default_handler()
            transformers.utils.logging.enable_explicit_format()
      logger.info("Training/evaluation parameters %s"   ,					__UpperCamelCase )

      # Set seed
      set_seed(training_args.seed )

      try:
            SCREAMING_SNAKE_CASE_      =     processors[data_args.task_name]()
            SCREAMING_SNAKE_CASE_      =     processor.get_labels()
            SCREAMING_SNAKE_CASE_      =     len(__UpperCamelCase )
      except KeyError:
            raise ValueError("Task not found: %s" % (data_args.task_name) )

      # Load pretrained model and tokenizer
      #
      # Distributed training:
      # The .from_pretrained methods guarantee that only one local process can concurrently
      # download model & vocab.

      SCREAMING_SNAKE_CASE_      =     AutoConfig.from_pretrained(
          model_args.config_name if model_args.config_name else model_args.model_name_or_path   ,					num_labels=__UpperCamelCase   ,					finetuning_task=data_args.task_name   ,					cache_dir=model_args.cache_dir   ,					)
      SCREAMING_SNAKE_CASE_      =     AutoTokenizer.from_pretrained(
          model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path   ,					cache_dir=model_args.cache_dir   ,					)
      SCREAMING_SNAKE_CASE_      =     AutoModelForMultipleChoice.from_pretrained(
          model_args.model_name_or_path   ,					from_tf=bool(".ckpt" in model_args.model_name_or_path )   ,					config=__UpperCamelCase   ,					cache_dir=model_args.cache_dir   ,					)

      # Get datasets
      SCREAMING_SNAKE_CASE_      =     (
          MultipleChoiceDataset(
              data_dir=data_args.data_dir   ,					tokenizer=__UpperCamelCase   ,					task=data_args.task_name   ,					max_seq_length=data_args.max_seq_length   ,					overwrite_cache=data_args.overwrite_cache   ,					mode=Split.train   ,					)
          if training_args.do_train
          else None
      )
      SCREAMING_SNAKE_CASE_      =     (
          MultipleChoiceDataset(
              data_dir=data_args.data_dir   ,					tokenizer=__UpperCamelCase   ,					task=data_args.task_name   ,					max_seq_length=data_args.max_seq_length   ,					overwrite_cache=data_args.overwrite_cache   ,					mode=Split.dev   ,					)
          if training_args.do_eval
          else None
      )

      def compute_metrics(__UpperCamelCase ) -> Dict:
            SCREAMING_SNAKE_CASE_      =     np.argmax(p.predictions   ,					axis=1 )
            return {"acc": simple_accuracy(__UpperCamelCase   ,					p.label_ids )}

      # Data collator
      SCREAMING_SNAKE_CASE_      =     DataCollatorWithPadding(__UpperCamelCase   ,					pad_to_multiple_of=8 ) if training_args.fpaa else None

      # Initialize our Trainer
      SCREAMING_SNAKE_CASE_      =     Trainer(
          model=__UpperCamelCase   ,					args=__UpperCamelCase   ,					train_dataset=__UpperCamelCase   ,					eval_dataset=__UpperCamelCase   ,					compute_metrics=__UpperCamelCase   ,					data_collator=__UpperCamelCase   ,					)

      # Training
      if training_args.do_train:
            trainer.train(
                model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path ) else None )
            trainer.save_model()
            # For convenience, we also re-save the tokenizer to the same directory,
            # so that you can share your model easily on huggingface.co/models =)
            if trainer.is_world_master():
                  tokenizer.save_pretrained(training_args.output_dir )

    # Evaluation
      SCREAMING_SNAKE_CASE_      =     {}
      if training_args.do_eval:
            logger.info("*** Evaluate ***" )

            SCREAMING_SNAKE_CASE_      =     trainer.evaluate()

            SCREAMING_SNAKE_CASE_      =     os.path.join(training_args.output_dir   ,					"eval_results.txt" )
            if trainer.is_world_master():
                  with open(__UpperCamelCase   ,					"w" ) as writer:
                        logger.info("***** Eval results *****" )
                        for key, value in result.items():
                              logger.info("  %s = %s"   ,					__UpperCamelCase   ,					__UpperCamelCase )
                              writer.write("%s = %s\n" % (key, value) )

                        results.update(__UpperCamelCase )

      return results


def    a__       (						__UpperCamelCase ):
      # For xla_spawn (TPUs)
      main()


if __name__ == "__main__":
      main()


