




'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import numpy as np
							import tensorflow as tf

							from transformers import (
							    TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
							    FlaubertConfig,
							    TFFlaubertForMultipleChoice,
							    TFFlaubertForQuestionAnsweringSimple,
							    TFFlaubertForSequenceClassification,
							    TFFlaubertForTokenClassification,
							    TFFlaubertModel,
							    TFFlaubertWithLMHeadModel,
							)






class 		a							:


				def __init__(						self       :   Tuple			,							lowercase_       :   Any			,							):
									snake_case_		     =							parent
									snake_case_		     =							13
									snake_case_		     =							7
									snake_case_		     =							True
									snake_case_		     =							True
									snake_case_		     =							True
									snake_case_		     =							True
									snake_case_		     =							True
									snake_case_		     =							False
									snake_case_		     =							False
									snake_case_		     =							False
									snake_case_		     =							2
									snake_case_		     =							99
									snake_case_		     =							0
									snake_case_		     =							32
									snake_case_		     =							2
									snake_case_		     =							4
									snake_case_		     =							0.1
									snake_case_		     =							0.1
									snake_case_		     =							512
									snake_case_		     =							16
									snake_case_		     =							2
									snake_case_		     =							0.02
									snake_case_		     =							3
									snake_case_		     =							4
									snake_case_		     =							'''last'''
									snake_case_		     =							True
									snake_case_		     =							None
									snake_case_		     =							0


				def     A_					(						self       :   Union[str, Any]   ):
									snake_case_		     =							ids_tensor([self.batch_size, self.seq_length]			,							self.vocab_size   )
									snake_case_		     =							random_attention_mask([self.batch_size, self.seq_length]			,							dtype=tf.floataa   )

									snake_case_		     =							None
									if self.use_input_lengths:
														snake_case_		     =							(
														    ids_tensor([self.batch_size]			,							vocab_size=2   ) + self.seq_length - 2
														)  # small variation of seq_length

									snake_case_		     =							None
									if self.use_token_type_ids:
														snake_case_		     =							ids_tensor([self.batch_size, self.seq_length]			,							self.n_langs   )

									snake_case_		     =							None
									snake_case_		     =							None
									snake_case_		     =							None
									if self.use_labels:
														snake_case_		     =							ids_tensor([self.batch_size]			,							self.type_sequence_label_size   )
														snake_case_		     =							ids_tensor([self.batch_size, self.seq_length]			,							self.num_labels   )
														snake_case_		     =							ids_tensor([self.batch_size]			,							2			,							dtype=tf.floataa   )
														snake_case_		     =							ids_tensor([self.batch_size]			,							self.num_choices   )

									snake_case_		     =							FlaubertConfig(
									    vocab_size=self.vocab_size			,							n_special=self.n_special			,							emb_dim=self.hidden_size			,							n_layers=self.num_hidden_layers			,							n_heads=self.num_attention_heads			,							dropout=self.hidden_dropout_prob			,							attention_dropout=self.attention_probs_dropout_prob			,							gelu_activation=self.gelu_activation			,							sinusoidal_embeddings=self.sinusoidal_embeddings			,							asm=self.asm			,							causal=self.causal			,							n_langs=self.n_langs			,							max_position_embeddings=self.max_position_embeddings			,							initializer_range=self.initializer_range			,							summary_type=self.summary_type			,							use_proj=self.use_proj			,							bos_token_id=self.bos_token_id			,							)

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_lengths,
									    sequence_labels,
									    token_labels,
									    is_impossible_labels,
									    choice_labels,
									    input_mask,
									)


				def     A_					(						self       :   List[Any]			,							lowercase_       :   Dict			,							lowercase_       :   Optional[int]			,							lowercase_       :   Any			,							lowercase_       :   List[Any]			,							lowercase_       :   Optional[int]			,							lowercase_       :   int			,							lowercase_       :   int			,							lowercase_       :   str			,							lowercase_       :   int			,							):
									snake_case_		     =							TFFlaubertModel(config=lowercase_   )
									snake_case_		     =							{'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
									snake_case_		     =							model(lowercase_   )

									snake_case_		     =							[input_ids, input_mask]
									snake_case_		     =							model(lowercase_   )
									self.parent.assertEqual(result.last_hidden_state.shape			,							(self.batch_size, self.seq_length, self.hidden_size)   )


				def     A_					(						self       :   List[str]			,							lowercase_       :   int			,							lowercase_       :   List[str]			,							lowercase_       :   Optional[Any]			,							lowercase_       :   str			,							lowercase_       :   Tuple			,							lowercase_       :   Tuple			,							lowercase_       :   Any			,							lowercase_       :   Dict			,							lowercase_       :   List[Any]			,							):
									snake_case_		     =							TFFlaubertWithLMHeadModel(lowercase_   )

									snake_case_		     =							{'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
									snake_case_		     =							model(lowercase_   )

									self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.seq_length, self.vocab_size)   )


				def     A_					(						self       :   str			,							lowercase_       :   Dict			,							lowercase_       :   Union[str, Any]			,							lowercase_       :   Dict			,							lowercase_       :   Any			,							lowercase_       :   str			,							lowercase_       :   List[str]			,							lowercase_       :   List[Any]			,							lowercase_       :   Union[str, Any]			,							lowercase_       :   str			,							):
									snake_case_		     =							TFFlaubertForQuestionAnsweringSimple(lowercase_   )

									snake_case_		     =							{'''input_ids''': input_ids, '''lengths''': input_lengths}

									snake_case_		     =							model(lowercase_   )

									self.parent.assertEqual(result.start_logits.shape			,							(self.batch_size, self.seq_length)   )
									self.parent.assertEqual(result.end_logits.shape			,							(self.batch_size, self.seq_length)   )


				def     A_					(						self       :   Optional[Any]			,							lowercase_       :   Any			,							lowercase_       :   int			,							lowercase_       :   List[Any]			,							lowercase_       :   Any			,							lowercase_       :   Optional[Any]			,							lowercase_       :   str			,							lowercase_       :   int			,							lowercase_       :   int			,							lowercase_       :   Dict			,							):
									snake_case_		     =							TFFlaubertForSequenceClassification(lowercase_   )

									snake_case_		     =							{'''input_ids''': input_ids, '''lengths''': input_lengths}

									snake_case_		     =							model(lowercase_   )

									self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.type_sequence_label_size)   )


				def     A_					(						self       :   List[Any]			,							lowercase_       :   Optional[Any]			,							lowercase_       :   Any			,							lowercase_       :   Any			,							lowercase_       :   List[Any]			,							lowercase_       :   List[Any]			,							lowercase_       :   Dict			,							lowercase_       :   int			,							lowercase_       :   Tuple			,							lowercase_       :   Union[str, Any]			,							):
									snake_case_		     =							self.num_labels
									snake_case_		     =							TFFlaubertForTokenClassification(config=lowercase_   )
									snake_case_		     =							{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
									snake_case_		     =							model(lowercase_   )
									self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.seq_length, self.num_labels)   )


				def     A_					(						self       :   Dict			,							lowercase_       :   Tuple			,							lowercase_       :   Optional[int]			,							lowercase_       :   Any			,							lowercase_       :   List[Any]			,							lowercase_       :   int			,							lowercase_       :   Tuple			,							lowercase_       :   List[str]			,							lowercase_       :   List[Any]			,							lowercase_       :   Any			,							):
									snake_case_		     =							self.num_choices
									snake_case_		     =							TFFlaubertForMultipleChoice(config=lowercase_   )
									snake_case_		     =							tf.tile(tf.expand_dims(lowercase_			,							1   )			,							(1, self.num_choices, 1)   )
									snake_case_		     =							tf.tile(tf.expand_dims(lowercase_			,							1   )			,							(1, self.num_choices, 1)   )
									snake_case_		     =							tf.tile(tf.expand_dims(lowercase_			,							1   )			,							(1, self.num_choices, 1)   )
									snake_case_		     =							{
									    '''input_ids''': multiple_choice_inputs_ids,
									    '''attention_mask''': multiple_choice_input_mask,
									    '''token_type_ids''': multiple_choice_token_type_ids,
									}
									snake_case_		     =							model(lowercase_   )
									self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.num_choices)   )


				def     A_					(						self       :   Dict   ):
									snake_case_		     =							self.prepare_config_and_inputs()
									(
									    (
									    snake_case_
									)     ,(
									    snake_case_
									)     ,(
									    snake_case_
									)     ,(
									    snake_case_
									)     ,(
									    snake_case_
									)     ,(
									    snake_case_
									)     ,(
									    snake_case_
									)     ,(
									    snake_case_
									)     ,(
									    snake_case_
									)     ,
									)		     =							config_and_inputs
									snake_case_		     =							{
									    '''input_ids''': input_ids,
									    '''token_type_ids''': token_type_ids,
									    '''langs''': token_type_ids,
									    '''lengths''': input_lengths,
									}
									return config, inputs_dict







@require_tf
class 		a							(  _lowerCamelCase  ,	_lowerCamelCase  ,	unittest.TestCase	):
				snake_case_           =						(
				    (
				        TFFlaubertModel,
				        TFFlaubertWithLMHeadModel,
				        TFFlaubertForSequenceClassification,
				        TFFlaubertForQuestionAnsweringSimple,
				        TFFlaubertForTokenClassification,
				        TFFlaubertForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				snake_case_           =						(
				    (TFFlaubertWithLMHeadModel,) if is_tf_available() else ()
				)  # TODO (PVP): Check other models whether language generation is also applicable
				snake_case_           =						(
				    {
				        "feature-extraction": TFFlaubertModel,
				        "fill-mask": TFFlaubertWithLMHeadModel,
				        "question-answering": TFFlaubertForQuestionAnsweringSimple,
				        "text-classification": TFFlaubertForSequenceClassification,
				        "token-classification": TFFlaubertForTokenClassification,
				        "zero-shot": TFFlaubertForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)
				snake_case_           =						False
				snake_case_           =						False


				def     A_					(						self       :   str			,							lowercase_       :   Optional[Any]			,							lowercase_       :   Optional[Any]			,							lowercase_       :   Union[str, Any]			,							lowercase_       :   Union[str, Any]			,							lowercase_       :   Optional[Any]   ):
									if (
									    pipeline_test_casse_name == "QAPipelineTests"
									    and tokenizer_name is not None
									    and not tokenizer_name.endswith('''Fast'''   )
									):
														# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
														# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
														# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
														return True

									return False


				def     A_					(						self       :   int   ):
									snake_case_		     =							TFFlaubertModelTester(self   )
									snake_case_		     =							ConfigTester(self			,							config_class=lowercase_			,							emb_dim=37   )


				def     A_					(						self       :   List[str]   ):
									self.config_tester.run_common_tests()


				def     A_					(						self       :   Optional[Any]   ):
									snake_case_		     =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_model(*lowercase_   )


				def     A_					(						self       :   Union[str, Any]   ):
									snake_case_		     =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_lm_head(*lowercase_   )


				def     A_					(						self       :   Tuple   ):
									snake_case_		     =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_qa(*lowercase_   )


				def     A_					(						self       :   Tuple   ):
									snake_case_		     =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_sequence_classif(*lowercase_   )


				def     A_					(						self       :   List[str]   ):
									snake_case_		     =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_for_token_classification(*lowercase_   )


				def     A_					(						self       :   Union[str, Any]   ):
									snake_case_		     =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_for_multiple_choice(*lowercase_   )


				@slow
				def     A_					(						self       :   Optional[int]   ):
									for model_name in TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
														snake_case_		     =							TFFlaubertModel.from_pretrained(lowercase_   )
														self.assertIsNotNone(lowercase_   )







@require_tf
@require_sentencepiece
@require_tokenizers
class 		a							(  unittest.TestCase	):


				@slow
				def     A_					(						self       :   int   ):
									snake_case_		     =							TFFlaubertModel.from_pretrained('''jplu/tf-flaubert-small-cased'''   )

									snake_case_		     =							tf.convert_to_tensor(
									    [[0, 158, 735, 2592, 1424, 6727, 82, 1]]			,							dtype=tf.intaa			,							)  # "J'aime flaubert !"

									snake_case_		     =							model(lowercase_   )[0]
									snake_case_		     =							tf.TensorShape((1, 8, 512)   )
									self.assertEqual(output.shape			,							lowercase_   )
									# compare the actual values for a slice.
									snake_case_		     =							tf.convert_to_tensor(
									    [
									        [
									            [-1.876_8773, -1.56_6555, 0.2707_2418],
									            [-1.692_0038, -0.587_3505, 1.932_9599],
									            [-2.956_3985, -1.699_3835, 1.797_2052],
									        ]
									    ]			,							dtype=tf.floataa			,							)

									self.assertTrue(np.allclose(output[:, :3, :3].numpy()			,							expected_slice.numpy()			,							atol=1e-4   )   )







'''simple docstring'''

import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.local_sgd import LocalSGD


########################################################################
# This is a fully working simple example to use Accelerate
# with LocalSGD, which is a method to synchronize model
# parameters every K batches. It is different, but complementary
# to gradient accumulation.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


a    :							str										=    16
a    :							Union[str, Any]										=    32


def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase = 16      )						->   Union[str, Any]:

					'''simple docstring'''

					snake_case_		     =							AutoTokenizer.from_pretrained('''bert-base-cased'''      )
					snake_case_		     =							load_dataset('''glue''', '''mrpc'''      )

					def tokenize_function(__UpperCAmelCase      ):
										# max_length=None => use the model max length (it's actually the default)
										snake_case_		     =							tokenizer(examples['''sentence1'''], examples['''sentence2'''], truncation=__UpperCAmelCase, max_length=__UpperCAmelCase      )
										return outputs

					# Apply the method we just defined to all the examples in all the splits of the dataset
					# starting with the main process first:
					with accelerator.main_process_first():
										snake_case_		     =							datasets.map(
										    __UpperCAmelCase, batched=__UpperCAmelCase, remove_columns=['''idx''', '''sentence1''', '''sentence2'''], )

					# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
					# transformers library
					snake_case_		     =							tokenized_datasets.rename_column('''label''', '''labels'''      )

					def collate_fn(__UpperCAmelCase      ):
										# On TPU it's best to pad everything to the same length or training will be very slow.
										snake_case_		     =							128 if accelerator.distributed_type == DistributedType.TPU else None
										# When using mixed precision we want round multiples of 8/16
										if accelerator.mixed_precision == "fp8":
															snake_case_		     =							16
										elif accelerator.mixed_precision != "no":
															snake_case_		     =							8
										else:
															snake_case_		     =							None

										return tokenizer.pad(
										    __UpperCAmelCase, padding='''longest''', max_length=__UpperCAmelCase, pad_to_multiple_of=__UpperCAmelCase, return_tensors='''pt''', )

					# Instantiate dataloaders.
					snake_case_		     =							DataLoader(
					    tokenized_datasets['''train'''], shuffle=__UpperCAmelCase, collate_fn=__UpperCAmelCase, batch_size=__UpperCAmelCase      )
					snake_case_		     =							DataLoader(
					    tokenized_datasets['''validation'''], shuffle=__UpperCAmelCase, collate_fn=__UpperCAmelCase, batch_size=__UpperCAmelCase      )

					return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
							from accelerate.test_utils.training import mocked_dataloaders

							a    :							Optional[Any]										=    mocked_dataloaders  # noqa: F811


def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase      )						->   Optional[Any]:

					'''simple docstring'''

					if os.environ.get('''TESTING_MOCKED_DATALOADERS''', __UpperCAmelCase      ) == "1":
										snake_case_		     =							2
					# New Code #
					snake_case_		     =							int(args.gradient_accumulation_steps      )
					snake_case_		     =							int(args.local_sgd_steps      )
					# Initialize accelerator
					snake_case_		     =							Accelerator(
					    cpu=args.cpu, mixed_precision=args.mixed_precision, gradient_accumulation_steps=__UpperCAmelCase      )
					if accelerator.distributed_type not in [DistributedType.NO, DistributedType.MULTI_CPU, DistributedType.MULTI_GPU]:
										raise NotImplementedError('''LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'''      )
					# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
					snake_case_		     =							config['''lr''']
					snake_case_		     =							int(config['''num_epochs''']      )
					snake_case_		     =							int(config['''seed''']      )
					snake_case_		     =							int(config['''batch_size''']      )

					snake_case_		     =							evaluate.load('''glue''', '''mrpc'''      )

					set_seed(__UpperCAmelCase      )
					snake_case_     ,snake_case_		     =							get_dataloaders(__UpperCAmelCase, __UpperCAmelCase      )
					# Instantiate the model (we build the model here so that the seed also control new weights initialization)
					snake_case_		     =							AutoModelForSequenceClassification.from_pretrained('''bert-base-cased''', return_dict=__UpperCAmelCase      )

					# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
					# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
					# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
					snake_case_		     =							model.to(accelerator.device      )

					# Instantiate optimizer
					snake_case_		     =							AdamW(params=model.parameters(), lr=__UpperCAmelCase      )

					# Instantiate scheduler
					snake_case_		     =							get_linear_schedule_with_warmup(
					    optimizer=__UpperCAmelCase, num_warmup_steps=100, num_training_steps=(len(__UpperCAmelCase      ) * num_epochs), )

					# Prepare everything
					# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
					# prepare method.
					snake_case_     ,snake_case_     ,snake_case_     ,snake_case_     ,snake_case_		     =							accelerator.prepare(
					    __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase      )

					# Now we train the model
					for epoch in range(__UpperCAmelCase      ):
										model.train()
										with LocalSGD(
										    accelerator=__UpperCAmelCase, model=__UpperCAmelCase, local_sgd_steps=__UpperCAmelCase, enabled=local_sgd_steps is not None      ) as local_sgd:
															for step, batch in enumerate(__UpperCAmelCase      ):
																				# We could avoid this line since we set the accelerator with `device_placement=True`.
																				batch.to(accelerator.device      )
																				# New code #
																				# We use the new `accumulate` context manager to perform gradient accumulation
																				# We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
																				with accelerator.accumulate(__UpperCAmelCase      ):
																									snake_case_		     =							model(**__UpperCAmelCase      )
																									snake_case_		     =							output.loss
																									accelerator.backward(__UpperCAmelCase      )
																									optimizer.step()
																									lr_scheduler.step()
																									optimizer.zero_grad()
																									# LocalSGD-specific line
																									local_sgd.step()

										model.eval()
										for step, batch in enumerate(__UpperCAmelCase      ):
															# We could avoid this line since we set the accelerator with `device_placement=True`.
															batch.to(accelerator.device      )
															with torch.no_grad():
																				snake_case_		     =							model(**__UpperCAmelCase      )
															snake_case_		     =							outputs.logits.argmax(dim=-1      )
															snake_case_     ,snake_case_		     =							accelerator.gather_for_metrics((predictions, batch['''labels'''])      )
															metric.add_batch(
															    predictions=__UpperCAmelCase, references=__UpperCAmelCase, )

										snake_case_		     =							metric.compute()
										# Use accelerator.print to print only on the main process.
										accelerator.print(F"epoch {epoch}:", __UpperCAmelCase      )


def 						__magic_name__       (      )						->   str:

					'''simple docstring'''

					snake_case_		     =							argparse.ArgumentParser(description='''Simple example of training script.'''      )
					parser.add_argument(
					    '''--mixed_precision''', type=__UpperCAmelCase, default=__UpperCAmelCase, choices=['''no''', '''fp16''', '''bf16''', '''fp8'''], help='''Whether to use mixed precision. Choose'''
					    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
					    '''and an Nvidia Ampere GPU.''', )
					# New Code #
					parser.add_argument(
					    '''--gradient_accumulation_steps''', type=__UpperCAmelCase, default=1, help='''The number of minibatches to be ran before gradients are accumulated.''', )
					parser.add_argument(
					    '''--local_sgd_steps''', type=__UpperCAmelCase, default=8, help='''Number of local SGD steps or None to disable local SGD'''      )
					parser.add_argument('''--cpu''', action='''store_true''', help='''If passed, will train on the CPU.'''      )
					snake_case_		     =							parser.parse_args()
					snake_case_		     =							{'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
					training_function(__UpperCAmelCase, __UpperCAmelCase      )


if __name__ == "__main__":
							main()

