



import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment

def 	_lowerCAmelCase (    A__    ,	A__=()    ,	A__=None    ,	A__="no"    ,	A__="29500"						):
      lowercase__  =	False
      lowercase__  =	False
      if any(key.startswith('KAGGLE'						) for key in os.environ.keys()						):
            lowercase__  =	True
      elif "IPython" in sys.modules:
            lowercase__  =	'google.colab' in str(sys.modules['IPython'].get_ipython()						)

      try:
            lowercase__  =	PrecisionType(mixed_precision.lower()						)
      except ValueError:
            raise ValueError(
                F'''Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.'''						)

      if (in_colab or in_kaggle) and (os.environ.get('TPU_NAME'    ,	A__						) is not None):
            # TPU launch
            import torch_xla.distributed.xla_multiprocessing as xmp

            if len(AcceleratorState._shared_state						) > 0:
                  raise ValueError(
                      'To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside '
                      'your training function. Restart your notebook and make sure no cells initializes an '
                      '`Accelerator`.'						)
            if num_processes is None:
                  lowercase__  =	8

            lowercase__  =	PrepareForLaunch(A__    ,	distributed_type='TPU'						)
            print(F'''Launching a training on {num_processes} TPU cores.'''						)
            xmp.spawn(A__    ,	args=A__    ,	nprocs=A__    ,	start_method='fork'						)
      elif in_colab:
            # No need for a distributed launch otherwise as it's either CPU or one GPU.
            if torch.cuda.is_available():
                  print('Launching training on one GPU.'						)
            else:
                  print('Launching training on one CPU.'						)
            function(*A__						)
      else:
            if num_processes is None:
                  raise ValueError(
                      'You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call.'						)

            if num_processes > 1:
                  # Multi-GPU launch
                  from torch.multiprocessing import start_processes
                  from torch.multiprocessing.spawn import ProcessRaisedException

                  if len(AcceleratorState._shared_state						) > 0:
                        raise ValueError(
                            'To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized '
                            'inside your training function. Restart your notebook and make sure no cells initializes an '
                            '`Accelerator`.'						)

                  if torch.cuda.is_initialized():
                        raise ValueError(
                            'To launch a multi-GPU training from your notebook, you need to avoid running any instruction '
                            'using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA '
                            'function.'						)

                  # torch.distributed will expect a few environment variable to be here. We set the ones common to each
                  # process here (the other ones will be set be the launcher).
                  with patch_environment(
                      world_size=A__    ,	master_addr='127.0.01'    ,	master_port=A__    ,	mixed_precision=A__						):
                        lowercase__  =	PrepareForLaunch(A__    ,	distributed_type='MULTI_GPU'						)
                        print(F'''Launching training on {num_processes} GPUs.'''						)
                        try:
                              start_processes(A__    ,	args=A__    ,	nprocs=A__    ,	start_method='fork'						)
                        except ProcessRaisedException as e:
                              if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
                                    raise RuntimeError(
                                        'CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. '
                                        'This likely stems from an outside import causing issues once the `notebook_launcher()` is called. '
                                        'Please review your imports and test them when running the `notebook_launcher()` to identify '
                                        'which one is problematic.'						) from e

            else:
                  # No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
                  if is_mps_available():
                        lowercase__  =	'1'
                        print('Launching training on MPS.'						)
                  elif torch.cuda.is_available():
                        print('Launching training on one GPU.'						)
                  else:
                        print('Launching training on CPU.'						)
                  function(*A__						)

def 	_lowerCAmelCase (    A__    ,	A__=()    ,	A__=2						):
      from torch.multiprocessing import start_processes

      with tempfile.NamedTemporaryFile() as tmp_file:
            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
            # process here (the other ones will be set be the launcher).
            with patch_environment(
                world_size=A__    ,	master_addr='127.0.01'    ,	master_port='29500'    ,	accelerate_mixed_precision='no'    ,	accelerate_debug_rdv_file=tmp_file.name    ,	accelerate_use_cpu='yes'    ,	):
                  lowercase__  =	PrepareForLaunch(A__    ,	debug=A__						)
                  start_processes(A__    ,	args=A__    ,	nprocs=A__    ,	start_method='fork'						)





import argparse
import logging
import sys
from unittest.mock import patch

import run_glue_deebert

from transformers.testing_utils import TestCasePlus, get_gpu_count, require_torch_non_multi_gpu, slow


logging.basicConfig(level=logging.DEBUG)

a__   :					Optional[Any] 						=      logging.getLogger()

def 	_lowerCAmelCase (    ):
      lowercase__  =	argparse.ArgumentParser()
      parser.add_argument('-f'						)
      lowercase__  =	parser.parse_args()
      return args.f



class    UpperCAmelCase__(				lowerCamelCase						):

   '''simple docstring'''




   def       UpperCAmelCase		(    self	:      List[str])						->					None:




         """simple docstring"""

         lowercase__  =	logging.StreamHandler(sys.stdout)
         logger.addHandler(lowerCAmelCase)


   def       UpperCAmelCase		(    self	:      List[str]      ,					lowerCAmelCase	:      Union[str, Any])						->					List[Any]:




         """simple docstring"""

         lowercase__  =	get_gpu_count()

         if n_gpu > 1:
               pass
               # XXX: doesn't quite work with n_gpu > 1 https://github.com/huggingface/transformers/issues/10560
               # script = f"{self.examples_dir_str}/research_projects/deebert/run_glue_deebert.py"
               # distributed_args = f"-m torch.distributed.launch --nproc_per_node={n_gpu} {script}".split()
               # cmd = [sys.executable] + distributed_args + args
               # execute_subprocess_async(cmd, env=self.get_env())
               # XXX: test the results - need to save them first into .json file
         else:
               args.insert(0      ,					'run_glue_deebert.py')
               with patch.object(lowerCAmelCase      ,					'argv'      ,					lowerCAmelCase):
                     lowercase__  =	run_glue_deebert.main()
                     for value in result.values():
                           self.assertGreaterEqual(lowerCAmelCase      ,					0.6_66)



   @slow
   @require_torch_non_multi_gpu
   def       UpperCAmelCase		(    self	:      Optional[Any])						->					Union[str, Any]:




         """simple docstring"""

         lowercase__  =	'\n            --model_type roberta\n            --model_name_or_path roberta-base\n            --task_name MRPC\n            --do_train\n            --do_eval\n            --do_lower_case\n            --data_dir ./tests/fixtures/tests_samples/MRPC/\n            --max_seq_length 128\n            --per_gpu_eval_batch_size=1\n            --per_gpu_train_batch_size=8\n            --learning_rate 2e-4\n            --num_train_epochs 3\n            --overwrite_output_dir\n            --seed 42\n            --output_dir ./examples/deebert/saved_models/roberta-base/MRPC/two_stage\n            --plot_data_dir ./examples/deebert/results/\n            --save_steps 0\n            --overwrite_cache\n            --eval_after_first_stage\n            '.split()
         self.run_and_check(lowerCAmelCase)

         lowercase__  =	'\n            --model_type roberta\n            --model_name_or_path ./examples/deebert/saved_models/roberta-base/MRPC/two_stage\n            --task_name MRPC\n            --do_eval\n            --do_lower_case\n            --data_dir ./tests/fixtures/tests_samples/MRPC/\n            --output_dir ./examples/deebert/saved_models/roberta-base/MRPC/two_stage\n            --plot_data_dir ./examples/deebert/results/\n            --max_seq_length 128\n            --eval_each_highway\n            --eval_highway\n            --overwrite_cache\n            --per_gpu_eval_batch_size=1\n            '.split()
         self.run_and_check(lowerCAmelCase)

         lowercase__  =	'\n            --model_type roberta\n            --model_name_or_path ./examples/deebert/saved_models/roberta-base/MRPC/two_stage\n            --task_name MRPC\n            --do_eval\n            --do_lower_case\n            --data_dir ./tests/fixtures/tests_samples/MRPC/\n            --output_dir ./examples/deebert/saved_models/roberta-base/MRPC/two_stage\n            --plot_data_dir ./examples/deebert/results/\n            --max_seq_length 128\n            --early_exit_entropy 0.1\n            --eval_highway\n            --overwrite_cache\n            --per_gpu_eval_batch_size=1\n            '.split()
         self.run_and_check(lowerCAmelCase)
