
'''simple docstring'''







import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase         =    get_tests_dir("""fixtures/test_sentencepiece_no_bos.model""")







@require_sentencepiece
@require_tokenizers
class UpperCAmelCase__	( lowercase__    ,				unittest.TestCase	):





  """simple docstring"""

  __UpperCAmelCase				:				Optional[Any]   =					PegasusTokenizer
  __UpperCAmelCase				:				int   =					PegasusTokenizerFast
  __UpperCAmelCase				:				Tuple   =					True
  __UpperCAmelCase				:				Optional[Any]   =					True


  def   __lowercase		(		self      :     Any       ):


     '''simple docstring'''






     super().setUp()

     # We have a SentencePiece fixture for testing
     _a				:	Optional[int]			=   PegasusTokenizer(_a       )
     tokenizer.save_pretrained(self.tmpdirname       )


  @cached_property
  def   __lowercase		(		self      :     Optional[int]       ):


     '''simple docstring'''






     return PegasusTokenizer.from_pretrained('google/pegasus-large'       )


  def   __lowercase		(		self      :     str							,**_a      :     str       ):


     '''simple docstring'''






     return PegasusTokenizer.from_pretrained(self.tmpdirname							,**_a       )


  def   __lowercase		(		self      :     Union[str, Any]							,_a      :     Dict       ):


     '''simple docstring'''






     return ("This is a test", "This is a test")


  def   __lowercase		(		self      :     List[Any]       ):


     '''simple docstring'''






     _a				:	Any			=   '</s>'
     _a				:	Optional[int]			=   1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a       )							,_a       )
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a       )							,_a       )


  def   __lowercase		(		self      :     Tuple       ):


     '''simple docstring'''






     _a				:	List[str]			=   list(self.get_tokenizer().get_vocab().keys()       )

     self.assertEqual(vocab_keys[0]							,'<pad>'       )
     self.assertEqual(vocab_keys[1]							,'</s>'       )
     self.assertEqual(vocab_keys[-1]							,'v'       )
     self.assertEqual(len(_a       )							,1103       )


  def   __lowercase		(		self      :     Dict       ):


     '''simple docstring'''






     self.assertEqual(self.get_tokenizer().vocab_size							,1103       )


  def   __lowercase		(		self      :     Any       ):


     '''simple docstring'''






     _a				:	Union[str, Any]			=   self.rust_tokenizer_class.from_pretrained(self.tmpdirname       )
     _a				:	Tuple			=   self.tokenizer_class.from_pretrained(self.tmpdirname       )
     _a				:	Union[str, Any]			=   (
         'Let\'s see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important'
         ' </s> <pad> <pad> <pad>'
     )
     _a				:	Any			=   rust_tokenizer([raw_input_str]							,return_tensors=_a							,add_special_tokens=_a       ).input_ids[0]
     _a				:	List[str]			=   py_tokenizer([raw_input_str]							,return_tensors=_a							,add_special_tokens=_a       ).input_ids[0]
     self.assertListEqual(_a							,_a       )


  def   __lowercase		(		self      :     str       ):


     '''simple docstring'''






     _a				:	Union[str, Any]			=   self._large_tokenizer
     # <mask_1> masks whole sentence while <mask_2> masks single word
     _a				:	List[Any]			=   '<mask_1> To ensure a <mask_2> flow of bank resolutions.'
     _a				:	List[Any]			=   [2, 413, 615, 114, 3, 1971, 113, 1679, 1_0710, 107, 1]
     _a				:	Union[str, Any]			=   tokenizer([raw_input_str]							,return_tensors=_a       ).input_ids[0]
     self.assertListEqual(_a							,_a       )


  def   __lowercase		(		self      :     Any       ):


     '''simple docstring'''






     _a				:	str			=   self._large_tokenizer
     # The tracebacks for the following asserts are **better** without messages or self.assertEqual
     assert tokenizer.vocab_size == 9_6103
     assert tokenizer.pad_token_id == 0
     assert tokenizer.eos_token_id == 1
     assert tokenizer.offset == 103
     assert tokenizer.unk_token_id == tokenizer.offset + 2 == 105
     assert tokenizer.unk_token == "<unk>"
     assert tokenizer.model_max_length == 1024
     _a				:	Any			=   'To ensure a smooth flow of bank resolutions.'
     _a				:	int			=   [413, 615, 114, 2291, 1971, 113, 1679, 1_0710, 107, 1]
     _a				:	Union[str, Any]			=   tokenizer([raw_input_str]							,return_tensors=_a       ).input_ids[0]
     self.assertListEqual(_a							,_a       )
     assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]       ) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]


  @require_torch
  def   __lowercase		(		self      :     str       ):


     '''simple docstring'''






     _a				:	Tuple			=   ['This is going to be way too long.' * 150, 'short example']
     _a				:	List[str]			=   ['not super long but more than 5 tokens', 'tiny']
     _a				:	str			=   self._large_tokenizer(_a							,padding=_a							,truncation=_a							,return_tensors='pt'       )
     _a				:	int			=   self._large_tokenizer(
         text_target=_a							,max_length=5							,padding=_a							,truncation=_a							,return_tensors='pt'       )

     assert batch.input_ids.shape == (2, 1024)
     assert batch.attention_mask.shape == (2, 1024)
     assert targets["input_ids"].shape == (2, 5)
     assert len(_a       ) == 2  # input_ids, attention_mask.


  @slow
  def   __lowercase		(		self      :     List[Any]       ):


     '''simple docstring'''






     _a				:	Tuple			=   {'input_ids': [[3_8979, 143, 1_8485, 606, 130, 2_6669, 8_7686, 121, 5_4189, 1129, 111, 2_6669, 8_7686, 121, 9114, 1_4787, 121, 1_3249, 158, 592, 956, 121, 1_4621, 3_1576, 143, 6_2613, 108, 9688, 930, 4_3430, 1_1562, 6_2613, 304, 108, 1_1443, 897, 108, 9314, 1_7415, 6_3399, 108, 1_1443, 7614, 1_8316, 118, 4284, 7148, 1_2430, 143, 1400, 2_5703, 158, 111, 4284, 7148, 1_1772, 143, 2_1297, 1064, 158, 122, 204, 3506, 1754, 1133, 1_4787, 1581, 115, 3_3224, 4482, 111, 1355, 110, 2_9173, 317, 5_0833, 108, 2_0147, 9_4665, 111, 7_7198, 107, 1], [110, 6_2613, 117, 638, 112, 1133, 121, 2_0098, 1355, 7_9050, 1_3872, 135, 1596, 5_3541, 1352, 141, 1_3039, 5542, 124, 302, 518, 111, 268, 2956, 115, 149, 4427, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [139, 1235, 2799, 1_8289, 1_7780, 204, 109, 9474, 1296, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=_a							,model_name='google/bigbird-pegasus-large-arxiv'							,revision='ba85d0851d708441f91440d509690f1ab6353415'							,)







@require_sentencepiece
@require_tokenizers
class UpperCAmelCase__	( lowercase__    ,				unittest.TestCase	):





  """simple docstring"""

  __UpperCAmelCase				:				List[str]   =					PegasusTokenizer
  __UpperCAmelCase				:				Optional[Any]   =					PegasusTokenizerFast
  __UpperCAmelCase				:				List[Any]   =					True
  __UpperCAmelCase				:				Any   =					True


  def   __lowercase		(		self      :     List[Any]       ):


     '''simple docstring'''






     super().setUp()

     # We have a SentencePiece fixture for testing
     _a				:	Tuple			=   PegasusTokenizer(_a							,offset=0							,mask_token_sent=_a							,mask_token='[MASK]'       )
     tokenizer.save_pretrained(self.tmpdirname       )


  @cached_property
  def   __lowercase		(		self      :     List[Any]       ):


     '''simple docstring'''






     return PegasusTokenizer.from_pretrained('google/bigbird-pegasus-large-arxiv'       )


  def   __lowercase		(		self      :     List[Any]							,**_a      :     Optional[Any]       ):


     '''simple docstring'''






     return PegasusTokenizer.from_pretrained(self.tmpdirname							,**_a       )


  def   __lowercase		(		self      :     Dict							,_a      :     Union[str, Any]       ):


     '''simple docstring'''






     return ("This is a test", "This is a test")


  def   __lowercase		(		self      :     List[Any]       ):


     '''simple docstring'''






     _a				:	str			=   self.rust_tokenizer_class.from_pretrained(self.tmpdirname       )
     _a				:	List[str]			=   self.tokenizer_class.from_pretrained(self.tmpdirname       )
     _a				:	Optional[int]			=   (
         'Let\'s see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>'
         ' <pad> <pad> <pad>'
     )
     _a				:	int			=   rust_tokenizer([raw_input_str]							,return_tensors=_a							,add_special_tokens=_a       ).input_ids[0]
     _a				:	List[Any]			=   py_tokenizer([raw_input_str]							,return_tensors=_a							,add_special_tokens=_a       ).input_ids[0]
     self.assertListEqual(_a							,_a       )


  @require_torch
  def   __lowercase		(		self      :     List[Any]       ):


     '''simple docstring'''






     _a				:	Tuple			=   ['This is going to be way too long.' * 1000, 'short example']
     _a				:	Optional[Any]			=   ['not super long but more than 5 tokens', 'tiny']
     _a				:	int			=   self._large_tokenizer(_a							,padding=_a							,truncation=_a							,return_tensors='pt'       )
     _a				:	str			=   self._large_tokenizer(
         text_target=_a							,max_length=5							,padding=_a							,truncation=_a							,return_tensors='pt'       )

     assert batch.input_ids.shape == (2, 4096)
     assert batch.attention_mask.shape == (2, 4096)
     assert targets["input_ids"].shape == (2, 5)
     assert len(_a       ) == 2  # input_ids, attention_mask.


  def   __lowercase		(		self      :     Union[str, Any]       ):


     '''simple docstring'''






     _a				:	Optional[int]			=   (
         'This is an example string that is used to test the original TF implementation against the HF'
         ' implementation'
     )

     _a				:	Optional[Any]			=   self._large_tokenizer(_a       ).input_ids

     self.assertListEqual(
         _a							,[182, 117, 142, 587, 4211, 120, 117, 263, 112, 804, 109, 856, 2_5016, 3137, 464, 109, 2_6955, 3137, 1]							,)





'''simple docstring'''







import argparse
from typing import List

import evaluate
import numpy as np
import torch
from datasets import DatasetDict, load_dataset

# New Code #
# We'll be using StratifiedKFold for this example
from sklearn.model_selection import StratifiedKFold
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to perform Cross Validation,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__lowerCAmelCase         =    1_6
__lowerCAmelCase         =    3_2





def 					UpperCAmelCase_		(__a      :    Accelerator ,	__a      :    DatasetDict ,	__a      :    List[int] ,	__a      :    List[int] ,	__a      :    int = 1_6 ):






   """simple docstring"""

   _a				:	Union[str, Any]			=   AutoTokenizer.from_pretrained('bert-base-cased' )
   _a				:	str			=   DatasetDict(
       {
           'train': dataset['train'].select(__a ),
           'validation': dataset['train'].select(__a ),
           'test': dataset['validation'],
       } )

   def tokenize_function(__a      :    List[Any] ):
      # max_length=None => use the model max length (it's actually the default)
      _a				:	Optional[int]			=   tokenizer(examples['sentence1'] ,	examples['sentence2'] ,	truncation=__a ,	max_length=__a )
      return outputs

   # Apply the method we just defined to all the examples in all the splits of the dataset
   # starting with the main process first:
   with accelerator.main_process_first():
      _a				:	List[str]			=   datasets.map(
          __a ,	batched=__a ,	remove_columns=['idx', 'sentence1', 'sentence2'] ,	)

   # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
   # transformers library
   _a				:	List[Any]			=   tokenized_datasets.rename_column('label' ,	'labels' )

   def collate_fn(__a      :    int ):
      # On TPU it's best to pad everything to the same length or training will be very slow.
      _a				:	Dict			=   1_2_8 if accelerator.distributed_type == DistributedType.TPU else None
      # When using mixed precision we want round multiples of 8/16
      if accelerator.mixed_precision == "fp8":
         _a				:	Tuple			=   1_6
      elif accelerator.mixed_precision != "no":
         _a				:	List[Any]			=   8
      else:
         _a				:	List[Any]			=   None

      return tokenizer.pad(
          __a ,	padding='longest' ,	max_length=__a ,	pad_to_multiple_of=__a ,	return_tensors='pt' ,	)

   # Instantiate dataloaders.
   _a				:	Any			=   DataLoader(
       tokenized_datasets['train'] ,	shuffle=__a ,	collate_fn=__a ,	batch_size=__a )
   _a				:	Optional[int]			=   DataLoader(
       tokenized_datasets['validation'] ,	shuffle=__a ,	collate_fn=__a ,	batch_size=__a )

   _a				:	Optional[Any]			=   DataLoader(
       tokenized_datasets['test'] ,	shuffle=__a ,	collate_fn=__a ,	batch_size=__a )

   return train_dataloader, eval_dataloader, test_dataloader





def 					UpperCAmelCase_		(__a      :    Any ,	__a      :    Union[str, Any] ):






   """simple docstring"""

   _a				:	Dict			=   []
   # Download the dataset
   _a				:	Tuple			=   load_dataset('glue' ,	'mrpc' )
   # Create our splits
   _a				:	Union[str, Any]			=   StratifiedKFold(n_splits=int(args.num_folds ) )
   # Initialize accelerator
   _a				:	Any			=   Accelerator(cpu=args.cpu ,	mixed_precision=args.mixed_precision )
   # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
   _a				:	Optional[Any]			=   config['lr']
   _a				:	Optional[int]			=   int(config['num_epochs'] )
   _a				:	Dict			=   int(config['seed'] )
   _a				:	Dict			=   int(config['batch_size'] )

   _a				:	Optional[int]			=   evaluate.load('glue' ,	'mrpc' )

   # If the batch size is too big we use gradient accumulation
   _a				:	List[Any]			=   1
   if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
      _a				:	Any			=   batch_size // MAX_GPU_BATCH_SIZE
      _a				:	List[str]			=   MAX_GPU_BATCH_SIZE

   set_seed(__a )

   # New Code #
   # Create our folds:
   _a				:	int			=   kfold.split(np.zeros(datasets['train'].num_rows ) ,	datasets['train']['label'] )
   _a				:	Any			=   []
   # Iterate over them
   for i, (train_idxs, valid_idxs) in enumerate(__a ):
      _a,  _a,  _a				:	Optional[Any]			=   get_fold_dataloaders(
          __a ,	__a ,	__a ,	__a ,	)
      # Instantiate the model (we build the model here so that the seed also control new weights initialization)
      _a				:	Dict			=   AutoModelForSequenceClassification.from_pretrained('bert-base-cased' ,	return_dict=__a )

      # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
      # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
      # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
      _a				:	List[Any]			=   model.to(accelerator.device )

      # Instantiate optimizer
      _a				:	List[str]			=   AdamW(params=model.parameters() ,	lr=__a )

      # Instantiate scheduler
      _a				:	List[Any]			=   get_linear_schedule_with_warmup(
          optimizer=__a ,	num_warmup_steps=1_0_0 ,	num_training_steps=(len(__a ) * num_epochs) // gradient_accumulation_steps ,	)

      # Prepare everything
      # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
      # prepare method.
      _a,  _a,  _a,  _a,  _a				:	Union[str, Any]			=   accelerator.prepare(
          __a ,	__a ,	__a ,	__a ,	__a )

      # Now we train the model
      for epoch in range(__a ):
         model.train()
         for step, batch in enumerate(__a ):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device )
            _a				:	Dict			=   model(**__a )
            _a				:	int			=   outputs.loss
            _a				:	Any			=   loss / gradient_accumulation_steps
            accelerator.backward(__a )
            if step % gradient_accumulation_steps == 0:
               optimizer.step()
               lr_scheduler.step()
               optimizer.zero_grad()

         model.eval()
         for step, batch in enumerate(__a ):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device )
            with torch.no_grad():
               _a				:	Union[str, Any]			=   model(**__a )
            _a				:	Tuple			=   outputs.logits.argmax(dim=-1 )
            _a,  _a				:	Any			=   accelerator.gather_for_metrics((predictions, batch['labels']) )
            metric.add_batch(
                predictions=__a ,	references=__a ,	)

         _a				:	List[Any]			=   metric.compute()
         # Use accelerator.print to print only on the main process.
         accelerator.print(f"""epoch {epoch}:""" ,	__a )

      # New Code #
      # We also run predictions on the test set at the very end
      _a				:	Any			=   []
      for step, batch in enumerate(__a ):
         # We could avoid this line since we set the accelerator with `device_placement=True`.
         batch.to(accelerator.device )
         with torch.no_grad():
            _a				:	Tuple			=   model(**__a )
         _a				:	Dict			=   outputs.logits
         _a,  _a				:	Optional[int]			=   accelerator.gather_for_metrics((predictions, batch['labels']) )
         fold_predictions.append(predictions.cpu() )
         if i == 0:
            # We need all of the test predictions
            test_references.append(references.cpu() )
        # Use accelerator.print to print only on the main process.
      test_predictions.append(torch.cat(__a ,	dim=0 ) )
      # We now need to release all our memory and get rid of the current model, optimizer, etc
      accelerator.free_memory()
   # New Code #
   # Finally we check the accuracy of our folded results:
   _a				:	Dict			=   torch.cat(__a ,	dim=0 )
   _a				:	Any			=   torch.stack(__a ,	dim=0 ).sum(dim=0 ).div(int(args.num_folds ) ).argmax(dim=-1 )
   _a				:	str			=   metric.compute(predictions=__a ,	references=__a )
   accelerator.print('Average test metrics from all folds:' ,	__a )





def 					UpperCAmelCase_		():






   """simple docstring"""

   _a				:	Any			=   argparse.ArgumentParser(description='Simple example of training script.' )
   parser.add_argument(
       '--mixed_precision' ,	type=__a ,	default=__a ,	choices=['no', 'fp16', 'bf16', 'fp8'] ,	help='Whether to use mixed precision. Choose'
       'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
       'and an Nvidia Ampere GPU.' ,	)
   parser.add_argument('--cpu' ,	action='store_true' ,	help='If passed, will train on the CPU.' )
   # New Code #
   parser.add_argument('--num_folds' ,	type=__a ,	default=3 ,	help='The number of splits to perform across the dataset' )
   _a				:	Any			=   parser.parse_args()
   _a				:	int			=   {'lr': 2e-5, 'num_epochs': 3, 'seed': 4_2, 'batch_size': 1_6}
   training_function(__a ,	__a )


if __name__ == "__main__":
  main()



