






'''simple docstring'''
import numpy as np
from scipy.spatial.distance import cdist
from sklearn.metrics import fa_score

import datasets


UpperCamelCase__			=      '\\n    @inproceedings{kakwani2020indicnlpsuite,\n    title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}},\n    author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar},\n    year={2020},\n    booktitle={Findings of EMNLP},\n}\n'

UpperCamelCase__			=      '\\n    IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide\n    variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te.\n'

UpperCamelCase__			=      '\nCompute IndicGLUE evaluation metric associated to each IndicGLUE dataset.\nArgs:\n    predictions: list of predictions to score (as int64),\n        except for \'cvit-mkb-clsr\' where each prediction is a vector (of float32).\n    references: list of ground truth labels corresponding to the predictions (as int64),\n        except for \'cvit-mkb-clsr\' where each reference is a vector (of float32).\nReturns: depending on the IndicGLUE subset, one or several of:\n    "accuracy": Accuracy\n    "f1": F1 score\n    "precision": Precision@10\nExamples:\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wnli\')  # \'wnli\' or any of ["copa", "sna", "csqa", "wstp", "inltkh", "bbca", "iitp-mr", "iitp-pr", "actsa-sc", "md"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wiki-ner\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'cvit-mkb-clsr\')\n    >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'precision@10\': 1.0}\n\n'




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase				):


   """simple docstring"""




   return float((preds == labels).mean()				)




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase				):


   """simple docstring"""




   lowercase_						:     Union[str, Any]							     =  simple_accuracy(_UpperCamelCase    ,		_UpperCamelCase				)
   lowercase_						:     List[Any]							     =  float(fa_score(y_true=_UpperCamelCase    ,		y_pred=_UpperCamelCase				)				)
   return {
       "accuracy": acc,
       "f1": fa,
   }




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase				):


   """simple docstring"""




   lowercase_						:     Optional[int]							     =  np.array(_UpperCamelCase				)
   lowercase_						:     Union[str, Any]							     =  np.array(_UpperCamelCase				)
   lowercase_						:     Union[str, Any]							     =  en_sentvecs.shape[0]

   # mean centering
   lowercase_						:     List[str]							     =  en_sentvecs - np.mean(_UpperCamelCase    ,		axis=0				)
   lowercase_						:     Dict							     =  in_sentvecs - np.mean(_UpperCamelCase    ,		axis=0				)

   lowercase_						:     int							     =  cdist(_UpperCamelCase    ,		_UpperCamelCase    ,		"cosine"				)
   lowercase_						:     str							     =  np.array(range(_UpperCamelCase				)				)
   lowercase_						:     Optional[int]							     =  sim.argsort(axis=1				)[:, :10]
   lowercase_						:     Any							     =  np.any(preds == actual[:, None]    ,		axis=1				)
   return float(matches.mean()				)





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,       _KWARGS_DESCRIPTION      )
class 							_UpperCAmelCase    (		datasets.Metric      ):




    def 					lowerCAmelCase__					(      self	:							Any      ):






       '''simple docstring'''






       if self.config_name not in [
           "wnli",
           "copa",
           "sna",
           "csqa",
           "wstp",
           "inltkh",
           "bbca",
           "cvit-mkb-clsr",
           "iitp-mr",
           "iitp-pr",
           "actsa-sc",
           "md",
           "wiki-ner",
       ]:
          raise KeyError(
              "You should supply a configuration name selected in "
              "[\"wnli\", \"copa\", \"sna\", \"csqa\", \"wstp\", \"inltkh\", \"bbca\", "
              "\"cvit-mkb-clsr\", \"iitp-mr\", \"iitp-pr\", \"actsa-sc\", \"md\", "
              "\"wiki-ner\"]"      )
       return datasets.MetricInfo(
           description=_DESCRIPTION	,  citation=_CITATION	,  inputs_description=_KWARGS_DESCRIPTION	,  features=datasets.Features(
               {
                   "predictions": datasets.Value("int64"      )
                   if self.config_name != "cvit-mkb-clsr"
                   else datasets.Sequence(datasets.Value("float32"      )      ),
                   "references": datasets.Value("int64"      )
                   if self.config_name != "cvit-mkb-clsr"
                   else datasets.Sequence(datasets.Value("float32"      )      ),
               }      )	,  codebase_urls=[]	,  reference_urls=[]	,  format="numpy" if self.config_name != "cvit-mkb-clsr" else None	,  )





    def 					lowerCAmelCase__					(      self	:							Optional[Any]	,  a	:							Optional[int]	,  a	:							int      ):






       '''simple docstring'''






       if self.config_name == "cvit-mkb-clsr":
          return {"precision@10": precision_at_aa(a	,  a      )}
       elif self.config_name in ["wiki-ner"]:
          return acc_and_fa(a	,  a      )
       elif self.config_name in [
           "wnli",
           "copa",
           "sna",
           "csqa",
           "wstp",
           "inltkh",
           "bbca",
           "iitp-mr",
           "iitp-pr",
           "actsa-sc",
           "md",
       ]:
          return {"accuracy": simple_accuracy(a	,  a      )}
       else:
          raise KeyError(
              "You should supply a configuration name selected in "
              "[\"wnli\", \"copa\", \"sna\", \"csqa\", \"wstp\", \"inltkh\", \"bbca\", "
              "\"cvit-mkb-clsr\", \"iitp-mr\", \"iitp-pr\", \"actsa-sc\", \"md\", "
              "\"wiki-ner\"]"      )












'''simple docstring'''
import inspect
import os
import unittest
from pathlib import Path

import torch

import accelerate
from accelerate.test_utils import execute_subprocess_async
from accelerate.test_utils.testing import run_command


class 							_UpperCAmelCase    (		unittest.TestCase      ):
    __lowerCamelCase: List[Any]									=     inspect.getfile(accelerate.test_utils      )
    __lowerCamelCase: List[Any]									=     os.path.sep.join(mod_file.split(os.path.sep      )[:-1] + ['scripts', 'test_cli.py']      )

    __lowerCamelCase: Optional[Any]									=     ['accelerate', 'launch']
    __lowerCamelCase: List[str]									=     Path.home() / '.cache/huggingface/accelerate'
    __lowerCamelCase: Dict									=     'default_config.yaml'
    __lowerCamelCase: Union[str, Any]									=     config_folder / config_file
    __lowerCamelCase: Tuple									=     config_folder / '_default_config.yaml'

    __lowerCamelCase: Union[str, Any]									=     Path('tests/test_configs'      )




    @classmethod
    def 					lowerCAmelCase__					(      cls	:							List[Any]      ):






       '''simple docstring'''






       if cls.config_path.is_file():
          cls.config_path.rename(cls.changed_path      )




    @classmethod
    def 					lowerCAmelCase__					(      cls	:							str      ):






       '''simple docstring'''






       if cls.changed_path.is_file():
          cls.changed_path.rename(cls.config_path      )




    def 					lowerCAmelCase__					(      self	:							Dict      ):






       '''simple docstring'''






       lowercase_						:     List[str]							     =  self.base_cmd
       if torch.cuda.is_available() and (torch.cuda.device_count() > 1):
          cmd += ["--multi_gpu"]
       execute_subprocess_async(cmd + [self.test_file_path]	,  env=os.environ.copy()      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       for config in sorted(self.test_config_path.glob("**/*.yaml"      )      ):
          with self.subTest(config_file=a      ):
             execute_subprocess_async(
                 self.base_cmd + ["--config_file", str(a      ), self.test_file_path]	,  env=os.environ.copy()      )





    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       execute_subprocess_async(["accelerate", "test"]	,  env=os.environ.copy()      )


class 							_UpperCAmelCase    (		unittest.TestCase      ):
    __lowerCamelCase: Tuple									=     'test-tpu'
    __lowerCamelCase: Dict									=     'us-central1-a'
    __lowerCamelCase: List[str]									=     'ls'
    __lowerCamelCase: Optional[int]									=     ['accelerate', 'tpu-config']
    __lowerCamelCase: Optional[Any]									=     'cd /usr/share'
    __lowerCamelCase: Any									=     'tests/test_samples/test_command_file.sh'
    __lowerCamelCase: Tuple									=     'Running gcloud compute tpus tpu-vm ssh'




    def 					lowerCAmelCase__					(      self	:							Tuple      ):






       '''simple docstring'''






       lowercase_						:     str							     =  run_command(
           self.cmd
           + ["--command", self.command, "--tpu_zone", self.tpu_zone, "--tpu_name", self.tpu_name, "--debug"]	,  return_stdout=a	,  )
       self.assertIn(
           f"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all"""	,  a	,  )




    def 					lowerCAmelCase__					(      self	:							Optional[Any]      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  run_command(
           self.cmd
           + [
               "--config_file",
               "tests/test_configs/0_12_0.yaml",
               "--command",
               self.command,
               "--tpu_zone",
               self.tpu_zone,
               "--tpu_name",
               self.tpu_name,
               "--debug",
           ]	,  return_stdout=a	,  )
       self.assertIn(
           f"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all"""	,  a	,  )




    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  run_command(
           self.cmd + ["--config_file", "tests/test_configs/latest.yaml", "--debug"]	,  return_stdout=a      )
       self.assertIn(
           f"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all"""	,  a	,  )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       lowercase_						:     Any							     =  run_command(
           self.cmd + ["--config_file", "tests/test_configs/latest.yaml", "--command", self.command, "--debug"]	,  return_stdout=a	,  )
       self.assertIn(
           f"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all"""	,  a	,  )




    def 					lowerCAmelCase__					(      self	:							Optional[Any]      ):






       '''simple docstring'''






       lowercase_						:     Optional[Any]							     =  run_command(
           self.cmd
           + [
               "--config_file",
               "tests/test_configs/latest.yaml",
               "--command",
               self.command,
               "--command",
               "echo \"Hello World\"",
               "--debug",
           ]	,  return_stdout=a	,  )
       self.assertIn(
           f"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls; echo \"Hello World\" --worker all"""	,  a	,  )




    def 					lowerCAmelCase__					(      self	:							Any      ):






       '''simple docstring'''






       lowercase_						:     Optional[Any]							     =  run_command(
           self.cmd
           + ["--config_file", "tests/test_configs/latest.yaml", "--command_file", self.command_file, "--debug"]	,  return_stdout=a	,  )
       self.assertIn(
           f"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all"""	,  a	,  )




    def 					lowerCAmelCase__					(      self	:							Dict      ):






       '''simple docstring'''






       lowercase_						:     List[Any]							     =  run_command(
           self.cmd
           + [
               "--config_file",
               "tests/test_configs/0_12_0.yaml",
               "--command_file",
               self.command_file,
               "--tpu_zone",
               self.tpu_zone,
               "--tpu_name",
               self.tpu_name,
               "--debug",
           ]	,  return_stdout=a	,  )
       self.assertIn(
           f"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all"""	,  a	,  )




    def 					lowerCAmelCase__					(      self	:							Dict      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  run_command(
           self.cmd + ["--config_file", "tests/test_configs/latest.yaml", "--install_accelerate", "--debug"]	,  return_stdout=a	,  )
       self.assertIn(
           f"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate -U; echo \"hello world\"; echo \"this is a second command\" --worker all"""	,  a	,  )





    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       lowercase_						:     Optional[Any]							     =  run_command(
           self.cmd
           + [
               "--config_file",
               "tests/test_configs/latest.yaml",
               "--install_accelerate",
               "--accelerate_version",
               "12.0.0",
               "--debug",
           ]	,  return_stdout=a	,  )
       self.assertIn(
           f"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate==12.0.0; echo \"hello world\"; echo \"this is a second command\" --worker all"""	,  a	,  )




