






'''simple docstring'''
import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


UpperCamelCase__			=      logging.getLogger(__name__)

require_version('pytorch_lightning>=1.0.4')

UpperCamelCase__			=      {
    'base': AutoModel,
    'sequence-classification': AutoModelForSequenceClassification,
    'question-answering': AutoModelForQuestionAnswering,
    'pretraining': AutoModelForPreTraining,
    'token-classification': AutoModelForTokenClassification,
    'language-modeling': AutoModelWithLMHead,
    'summarization': AutoModelForSeqaSeqLM,
    'translation': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
UpperCamelCase__			=      {
    'linear': get_linear_schedule_with_warmup,
    'cosine': get_cosine_schedule_with_warmup,
    'cosine_w_restarts': get_cosine_with_hard_restarts_schedule_with_warmup,
    'polynomial': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
UpperCamelCase__			=      sorted(arg_to_scheduler.keys())
UpperCamelCase__			=      '{' + ', '.join(arg_to_scheduler_choices) + '}'


class 							_UpperCAmelCase    (		pl.LightningModule      ):




    def __init__(      self	:							Optional[Any]	,  a	:							argparse.Namespace	,  a	:							Any=None	,  a	:							Union[str, Any]="base"	,  a	:							Dict=None	,  a	:							Optional[int]=None	,  a	:							List[Any]=None	,  **a	:							List[Any]	,  ):






       '''simple docstring'''






       super().__init__()
       # TODO: move to self.save_hyperparameters()
       # self.save_hyperparameters()
       # can also expand arguments into trainer signature for easier reading

       self.save_hyperparameters(a      )
       lowercase_						:     int							     =  0
       lowercase_						:     Optional[int]							     =  Path(self.hparams.output_dir      )
       lowercase_						:     Dict							     =  self.hparams.cache_dir if self.hparams.cache_dir else None
       if config is None:
          lowercase_						:     Union[str, Any]							     =  AutoConfig.from_pretrained(
              self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path	,  **({"num_labels": num_labels} if num_labels is not None else {})	,  cache_dir=a	,  **a	,  )
       else:
          lowercase_						:     PretrainedConfig							     =  config

       lowercase_						:     Any							     =  ("encoder_layerdrop", "decoder_layerdrop", "dropout", "attention_dropout")
       for p in extra_model_params:
          if getattr(self.hparams	,  a	,  a      ):
             assert hasattr(self.config	,  a      ), f"""model config doesn't have a `{p}` attribute"""
             setattr(self.config	,  a	,  getattr(self.hparams	,  a      )      )

       if tokenizer is None:
          lowercase_						:     Optional[Any]							     =  AutoTokenizer.from_pretrained(
              self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path	,  cache_dir=a	,  )
       else:
          lowercase_						:     PreTrainedTokenizer							     =  tokenizer
       lowercase_						:     int							     =  MODEL_MODES[mode]
       if model is None:
          lowercase_						:     List[Any]							     =  self.model_type.from_pretrained(
              self.hparams.model_name_or_path	,  from_tf=bool(".ckpt" in self.hparams.model_name_or_path      )	,  config=self.config	,  cache_dir=a	,  )
       else:
          lowercase_						:     List[Any]							     =  model




    def 					lowerCAmelCase__					(      self	:							int	,  *a	:							str	,  **a	:							Optional[Any]      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  self.model_type.from_pretrained(*a	,  **a      )




    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       lowercase_						:     str							     =  arg_to_scheduler[self.hparams.lr_scheduler]
       lowercase_						:     Union[str, Any]							     =  get_schedule_func(
           self.opt	,  num_warmup_steps=self.hparams.warmup_steps	,  num_training_steps=self.total_steps()      )
       lowercase_						:     int							     =  {"scheduler": scheduler, "interval": "step", "frequency": 1}
       return scheduler




    def 					lowerCAmelCase__					(      self	:							Any      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  self.model
       lowercase_						:     Union[str, Any]							     =  ["bias", "LayerNorm.weight"]
       lowercase_						:     int							     =  [
           {
               "params": [
                   p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay      )
               ],  # check this named paramters
               "weight_decay": self.hparams.weight_decay,
           },
           {
               "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay      )],
               "weight_decay": 0.0,
           },
       ]
       if self.hparams.adafactor:
          lowercase_						:     List[str]							     =  Adafactor(
              a	,  lr=self.hparams.learning_rate	,  scale_parameter=a	,  relative_step=a      )

       else:
          lowercase_						:     Tuple							     =  AdamW(
              a	,  lr=self.hparams.learning_rate	,  eps=self.hparams.adam_epsilon      )
       lowercase_						:     List[str]							     =  optimizer

       lowercase_						:     Optional[int]							     =  self.get_lr_scheduler()

       return [optimizer], [scheduler]




    def 					lowerCAmelCase__					(      self	:							Dict	,  a	:							int	,  a	:							Union[str, Any]      ):






       '''simple docstring'''






       return self.validation_step(a	,  a      )




    def 					lowerCAmelCase__					(      self	:							str	,  a	:							Optional[Any]      ):






       '''simple docstring'''






       return self.validation_end(a      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       lowercase_						:     Dict							     =  max(1	,  self.hparams.gpus      )  # TODO: consider num_tpu_cores
       lowercase_						:     Union[str, Any]							     =  self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
       return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs




    def 					lowerCAmelCase__					(      self	:							Any	,  a	:							int      ):






       '''simple docstring'''






       if stage == "test":
          lowercase_						:     Tuple							     =  len(self.test_dataloader().dataset      )
       else:
          lowercase_						:     int							     =  self.get_dataloader("train"	,  self.hparams.train_batch_size	,  shuffle=a      )
          lowercase_						:     str							     =  len(self.train_dataloader().dataset      )




    def 					lowerCAmelCase__					(      self	:							int	,  a	:							str	,  a	:							int	,  a	:							bool = False      ):






       '''simple docstring'''






       raise NotImplementedError("You must implement this for your task"      )




    def 					lowerCAmelCase__					(      self	:							Any      ):






       '''simple docstring'''






       return self.train_loader




    def 					lowerCAmelCase__					(      self	:							Any      ):






       '''simple docstring'''






       return self.get_dataloader("dev"	,  self.hparams.eval_batch_size	,  shuffle=a      )




    def 					lowerCAmelCase__					(      self	:							List[Any]      ):






       '''simple docstring'''






       return self.get_dataloader("test"	,  self.hparams.eval_batch_size	,  shuffle=a      )




    def 					lowerCAmelCase__					(      self	:							Optional[Any]	,  a	:							str      ):






       '''simple docstring'''






       return os.path.join(
           self.hparams.data_dir	,  "cached_{}_{}_{}".format(
               a	,  list(filter(a	,  self.hparams.model_name_or_path.split("/"      )      )      ).pop()	,  str(self.hparams.max_seq_length      )	,  )	,  )




    @pl.utilities.rank_zero_only
    def 					lowerCAmelCase__					(      self	:							List[Any]	,  a	:							Dict[str, Any]      ):






       '''simple docstring'''






       lowercase_						:     Optional[Any]							     =  self.output_dir.joinpath("best_tfmr"      )
       lowercase_						:     Optional[Any]							     =  self.step_count
       self.model.save_pretrained(a      )
       self.tokenizer.save_pretrained(a      )





    @staticmethod
    def 					lowerCAmelCase__					(      a	:							Union[str, Any]	,  a	:							Tuple      ):






       '''simple docstring'''






       parser.add_argument(
           "--model_name_or_path"	,  default=a	,  type=a	,  required=a	,  help="Path to pretrained model or model identifier from huggingface.co/models"	,  )
       parser.add_argument(
           "--config_name"	,  default=""	,  type=a	,  help="Pretrained config name or path if not the same as model_name"      )
       parser.add_argument(
           "--tokenizer_name"	,  default=a	,  type=a	,  help="Pretrained tokenizer name or path if not the same as model_name"	,  )
       parser.add_argument(
           "--cache_dir"	,  default=str(Path(a      ).parent / "test_run" / "cache"      )	,  type=a	,  help="Where do you want to store the pre-trained models downloaded from huggingface.co"	,  )
       parser.add_argument(
           "--encoder_layerdrop"	,  type=a	,  help="Encoder layer dropout probability (Optional). Goes into model.config"	,  )
       parser.add_argument(
           "--decoder_layerdrop"	,  type=a	,  help="Decoder layer dropout probability (Optional). Goes into model.config"	,  )
       parser.add_argument(
           "--dropout"	,  type=a	,  help="Dropout probability (Optional). Goes into model.config"	,  )
       parser.add_argument(
           "--attention_dropout"	,  type=a	,  help="Attention dropout probability (Optional). Goes into model.config"	,  )
       parser.add_argument("--learning_rate"	,  default=5e-5	,  type=a	,  help="The initial learning rate for Adam."      )
       parser.add_argument(
           "--lr_scheduler"	,  default="linear"	,  choices=a	,  metavar=a	,  type=a	,  help="Learning rate scheduler"	,  )
       parser.add_argument("--weight_decay"	,  default=0.0	,  type=a	,  help="Weight decay if we apply some."      )
       parser.add_argument("--adam_epsilon"	,  default=1e-8	,  type=a	,  help="Epsilon for Adam optimizer."      )
       parser.add_argument("--warmup_steps"	,  default=0	,  type=a	,  help="Linear warmup over warmup_steps."      )
       parser.add_argument("--num_workers"	,  default=4	,  type=a	,  help="kwarg passed to DataLoader"      )
       parser.add_argument("--num_train_epochs"	,  dest="max_epochs"	,  default=3	,  type=a      )
       parser.add_argument("--train_batch_size"	,  default=3_2	,  type=a      )
       parser.add_argument("--eval_batch_size"	,  default=3_2	,  type=a      )
       parser.add_argument("--adafactor"	,  action="store_true"      )


class 							_UpperCAmelCase    (		pl.Callback      ):




    def 					lowerCAmelCase__					(      self	:							Tuple	,  a	:							str	,  a	:							Tuple      ):






       '''simple docstring'''






       if (
           trainer.is_global_zero and trainer.global_rank == 0
       ):  # we initialize the retriever only on master worker with RAY. In new pytorch-lightning accelorators are removed.
          pl_module.model.rag.retriever.init_retrieval()  # better to use hook functions.


class 							_UpperCAmelCase    (		pl.Callback      ):




    def 					lowerCAmelCase__					(      self	:							List[Any]	,  a	:							Union[str, Any]	,  a	:							int      ):






       '''simple docstring'''






       for name, param in pl_module.model.rag.named_parameters():
          if param.grad is None:
             print(a      )


class 							_UpperCAmelCase    (		pl.Callback      ):




    def 					lowerCAmelCase__					(      self	:							Union[str, Any]	,  a	:							Optional[Any]	,  a	:							int      ):






       '''simple docstring'''






       lowercase_						:     List[Any]							     =  trainer.lr_schedulers[0]["scheduler"]
       lowercase_						:     Tuple							     =  {f"""lr_group_{i}""": lr for i, lr in enumerate(lr_scheduler.get_lr()      )}
       pl_module.logger.log_metrics(a      )




    def 					lowerCAmelCase__					(      self	:							int	,  a	:							pl.Trainer	,  a	:							pl.LightningModule      ):






       '''simple docstring'''






       rank_zero_info("***** Validation results *****"      )
       lowercase_						:     Any							     =  trainer.callback_metrics
       # Log results
       for key in sorted(a      ):
          if key not in ["log", "progress_bar"]:
             rank_zero_info("{} = {}\n".format(a	,  str(metrics[key]      )      )      )





    def 					lowerCAmelCase__					(      self	:							Dict	,  a	:							pl.Trainer	,  a	:							pl.LightningModule      ):






       '''simple docstring'''






       rank_zero_info("***** Test results *****"      )
       lowercase_						:     Tuple							     =  trainer.callback_metrics
       # Log and save results to file
       lowercase_						:     List[Any]							     =  os.path.join(pl_module.hparams.output_dir	,  "test_results.txt"      )
       with open(a	,  "w"      ) as writer:
          for key in sorted(a      ):
             if key not in ["log", "progress_bar"]:
                rank_zero_info("{} = {}\n".format(a	,  str(metrics[key]      )      )      )
                writer.write("{} = {}\n".format(a	,  str(metrics[key]      )      )      )




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase				):


   """simple docstring"""




   parser.add_argument(
       "--output_dir"    ,		default=str(Path(_UpperCamelCase				).parent / "test_run" / "model_checkpoints"				)    ,		type=_UpperCamelCase    ,		help="The output directory where the model predictions and checkpoints will be written."    ,		)
   parser.add_argument(
       "--fp16"    ,		action="store_true"    ,		help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"    ,		)

   parser.add_argument(
       "--fp16_opt_level"    ,		type=_UpperCamelCase    ,		default="O2"    ,		help=(
           "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
           "See details at https://nvidia.github.io/apex/amp.html"
       )    ,		)
   parser.add_argument("--n_tpu_cores"    ,		dest="tpu_cores"    ,		type=_UpperCamelCase				)
   parser.add_argument("--max_grad_norm"    ,		dest="gradient_clip_val"    ,		default=1.0    ,		type=_UpperCamelCase    ,		help="Max gradient norm"				)
   parser.add_argument("--do_train"    ,		action="store_true"    ,		help="Whether to run training."				)
   parser.add_argument("--do_predict"    ,		action="store_true"    ,		help="Whether to run predictions on the test set."				)
   parser.add_argument(
       "--gradient_accumulation_steps"    ,		dest="accumulate_grad_batches"    ,		type=_UpperCamelCase    ,		default=1    ,		help="Number of updates steps to accumulate before performing a backward/update pass."    ,		)
   parser.add_argument("--seed"    ,		type=_UpperCamelCase    ,		default=42    ,		help="random seed for initialization"				)
   parser.add_argument(
       "--data_dir"    ,		default=str(Path(_UpperCamelCase				).parent / "test_run" / "dummy-train-data"				)    ,		type=_UpperCamelCase    ,		help="The input data dir. Should contain the training files for the CoNLL-2003 NER task."    ,		)





def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase=None    ,		_UpperCamelCase=True    ,		_UpperCamelCase=[]    ,		_UpperCamelCase=None    ,		_UpperCamelCase=None    ,		**_UpperCamelCase    ,		):


   """simple docstring"""




   pl.seed_everything(args.seed				)

   # init model
   lowercase_						:     Dict							     =  Path(model.hparams.output_dir				)
   odir.mkdir(exist_ok=_UpperCamelCase				)

   # add custom checkpoints
   if checkpoint_callback is None:
      lowercase_						:     str							     =  pl.callbacks.ModelCheckpoint(
          filepath=args.output_dir    ,		prefix="checkpoint"    ,		monitor="val_loss"    ,		mode="min"    ,		save_top_k=1				)
   if early_stopping_callback:
      extra_callbacks.append(_UpperCamelCase				)
   if logging_callback is None:
      lowercase_						:     Any							     =  LoggingCallback()

   lowercase_						:     Dict							     =  {}

   if args.fpaa:
      lowercase_						:     Tuple							     =  16

   if args.gpus > 1:
      lowercase_						:     Any							     =  "auto"
      lowercase_						:     str							     =  "ddp"

   lowercase_						:     Optional[int]							     =  args.accumulate_grad_batches
   lowercase_						:     str							     =  None
   lowercase_						:     List[str]							     =  "auto"

   lowercase_						:     Dict							     =  pl.Trainer.from_argparse_args(
       _UpperCamelCase    ,		weights_summary=_UpperCamelCase    ,		callbacks=[logging_callback] + extra_callbacks + [InitCallback()] + [checkpoint_callback]    ,		logger=_UpperCamelCase    ,		val_check_interval=1    ,		num_sanity_val_steps=2    ,		**_UpperCamelCase    ,		)

   if args.do_train:
      trainer.fit(_UpperCamelCase				)

   else:
      print("RAG modeling tests with new set functions successfuly executed!"				)
   return trainer












'''simple docstring'''
import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
 from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
 from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
 from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


UpperCamelCase__			=      DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

UpperCamelCase__			=      'main'
# Default branch name
UpperCamelCase__			=      'f2c752cfc5c0ab6f4bdec59acea69eefbee381c2'
# One particular commit (not the top of `main`)
UpperCamelCase__			=      'aaaaaaa'
# This commit does not exist, so we should 404.

UpperCamelCase__			=      'd9e9f15bc825e4b2c9249e9578f884bbcb5e3684'
# Sha-1 of config.json on the top of `main`, for checking purposes
UpperCamelCase__			=      '4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3'




@contextlib.contextmanager
def 		__SCREAMING_SNAKE_CASE       (	):


   """simple docstring"""




   print("Welcome!"				)
   yield
   print("Bye!"				)




@contextlib.contextmanager
def 		__SCREAMING_SNAKE_CASE       (	):


   """simple docstring"""




   print("Bonjour!"				)
   yield
   print("Au revoir!"				)


class 							_UpperCAmelCase    (		unittest.TestCase      ):




    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       assert transformers.__spec__ is not None
       assert importlib.util.find_spec("transformers"      ) is not None


class 							_UpperCAmelCase    (		unittest.TestCase      ):




    @unittest.mock.patch("sys.stdout"	,  new_callable=io.StringIO      )
    def 					lowerCAmelCase__					(      self	:							str	,  a	:							List[str]      ):






       '''simple docstring'''






       with ContextManagers([]      ):
          print("Transformers are awesome!"      )
       # The print statement adds a new line at the end of the output
       self.assertEqual(mock_stdout.getvalue()	,  "Transformers are awesome!\n"      )




    @unittest.mock.patch("sys.stdout"	,  new_callable=io.StringIO      )
    def 					lowerCAmelCase__					(      self	:							List[str]	,  a	:							Optional[int]      ):






       '''simple docstring'''






       with ContextManagers([context_en()]      ):
          print("Transformers are awesome!"      )
       # The output should be wrapped with an English welcome and goodbye
       self.assertEqual(mock_stdout.getvalue()	,  "Welcome!\nTransformers are awesome!\nBye!\n"      )




    @unittest.mock.patch("sys.stdout"	,  new_callable=io.StringIO      )
    def 					lowerCAmelCase__					(      self	:							Tuple	,  a	:							List[Any]      ):






       '''simple docstring'''






       with ContextManagers([context_fr(), context_en()]      ):
          print("Transformers are awesome!"      )
       # The output should be wrapped with an English and French welcome and goodbye
       self.assertEqual(mock_stdout.getvalue()	,  "Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n"      )




    @require_torch
    def 					lowerCAmelCase__					(      self	:							Tuple      ):






       '''simple docstring'''






       self.assertEqual(find_labels(a      )	,  ["labels"]      )
       self.assertEqual(find_labels(a      )	,  ["labels", "next_sentence_label"]      )
       self.assertEqual(find_labels(a      )	,  ["start_positions", "end_positions"]      )


       class 							_UpperCAmelCase    (		snake_case      ):
           pass


       self.assertEqual(find_labels(a      )	,  ["labels"]      )




    @require_tf
    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       self.assertEqual(find_labels(a      )	,  ["labels"]      )
       self.assertEqual(find_labels(a      )	,  ["labels", "next_sentence_label"]      )
       self.assertEqual(find_labels(a      )	,  ["start_positions", "end_positions"]      )


       class 							_UpperCAmelCase    (		snake_case      ):
           pass


       self.assertEqual(find_labels(a      )	,  ["labels"]      )





    @require_flax
    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       self.assertEqual(find_labels(a      )	,  []      )
       self.assertEqual(find_labels(a      )	,  []      )
       self.assertEqual(find_labels(a      )	,  []      )


       class 							_UpperCAmelCase    (		snake_case      ):
           pass


       self.assertEqual(find_labels(a      )	,  []      )




