


'''simple docstring'''
import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.local_sgd import LocalSGD


########################################################################
# This is a fully working simple example to use Accelerate
# with LocalSGD, which is a method to synchronize model
# parameters every K batches. It is different, but complementary
# to gradient accumulation.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__snake_case							      =     16
__snake_case							      =     32


def 			a      (			__a ,					__a = 16   )  ->	List[Any]:



					'''simple docstring'''






					UpperCamelCase__   :Tuple												= AutoTokenizer.from_pretrained('''bert-base-cased'''   )
					UpperCamelCase__   :Optional[int]												= load_dataset('''glue''' ,					'''mrpc'''   )

					def tokenize_function(__a   ):
										# max_length=None => use the model max length (it's actually the default)
										UpperCamelCase__   :Union[str, Any]												= tokenizer(examples['''sentence1'''] ,					examples['''sentence2'''] ,					truncation=__a ,					max_length=__a   )
										return outputs

					# Apply the method we just defined to all the examples in all the splits of the dataset
					# starting with the main process first:
					with accelerator.main_process_first():
										UpperCamelCase__   :Optional[int]												= datasets.map(
										    __a ,					batched=__a ,					remove_columns=['''idx''', '''sentence1''', '''sentence2'''] ,					)

					# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
					# transformers library
					UpperCamelCase__   :List[str]												= tokenized_datasets.rename_column('''label''' ,					'''labels'''   )

					def collate_fn(__a   ):
										# On TPU it's best to pad everything to the same length or training will be very slow.
										UpperCamelCase__   :Any												= 128 if accelerator.distributed_type == DistributedType.TPU else None
										# When using mixed precision we want round multiples of 8/16
										if accelerator.mixed_precision == "fp8":
															UpperCamelCase__   :Tuple												= 16
										elif accelerator.mixed_precision != "no":
															UpperCamelCase__   :int												= 8
										else:
															UpperCamelCase__   :Dict												= None

										return tokenizer.pad(
										    __a ,					padding='''longest''' ,					max_length=__a ,					pad_to_multiple_of=__a ,					return_tensors='''pt''' ,					)

					# Instantiate dataloaders.
					UpperCamelCase__   :List[str]												= DataLoader(
					    tokenized_datasets['''train'''] ,					shuffle=__a ,					collate_fn=__a ,					batch_size=__a   )
					UpperCamelCase__   :Tuple												= DataLoader(
					    tokenized_datasets['''validation'''] ,					shuffle=__a ,					collate_fn=__a ,					batch_size=__a   )

					return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
		from accelerate.test_utils.training import mocked_dataloaders

		__snake_case							      =     mocked_dataloaders  # noqa: F811


def 			a      (			__a ,					__a   )  ->	Optional[int]:



					'''simple docstring'''






					if os.environ.get('''TESTING_MOCKED_DATALOADERS''' ,					__a   ) == "1":
										UpperCamelCase__   :List[str]												= 2
					# New Code #
					UpperCamelCase__   :Dict												= int(args.gradient_accumulation_steps   )
					UpperCamelCase__   :str												= int(args.local_sgd_steps   )
					# Initialize accelerator
					UpperCamelCase__   :List[Any]												= Accelerator(
					    cpu=args.cpu ,					mixed_precision=args.mixed_precision ,					gradient_accumulation_steps=__a   )
					if accelerator.distributed_type not in [DistributedType.NO, DistributedType.MULTI_CPU, DistributedType.MULTI_GPU]:
										raise NotImplementedError('''LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'''   )
					# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
					UpperCamelCase__   :Tuple												= config['''lr''']
					UpperCamelCase__   :str												= int(config['''num_epochs''']   )
					UpperCamelCase__   :int												= int(config['''seed''']   )
					UpperCamelCase__   :Optional[Any]												= int(config['''batch_size''']   )

					UpperCamelCase__   :List[str]												= evaluate.load('''glue''' ,					'''mrpc'''   )

					set_seed(__a   )
					UpperCamelCase__					,				UpperCamelCase__   :str												= get_dataloaders(__a ,					__a   )
					# Instantiate the model (we build the model here so that the seed also control new weights initialization)
					UpperCamelCase__   :List[Any]												= AutoModelForSequenceClassification.from_pretrained('''bert-base-cased''' ,					return_dict=__a   )

					# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
					# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
					# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
					UpperCamelCase__   :Optional[int]												= model.to(accelerator.device   )

					# Instantiate optimizer
					UpperCamelCase__   :Dict												= AdamW(params=model.parameters() ,					lr=__a   )

					# Instantiate scheduler
					UpperCamelCase__   :Union[str, Any]												= get_linear_schedule_with_warmup(
					    optimizer=__a ,					num_warmup_steps=100 ,					num_training_steps=(len(__a   ) * num_epochs) ,					)

					# Prepare everything
					# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
					# prepare method.
					UpperCamelCase__					,				UpperCamelCase__					,				UpperCamelCase__					,				UpperCamelCase__					,				UpperCamelCase__   :int												= accelerator.prepare(
					    __a ,					__a ,					__a ,					__a ,					__a   )

					# Now we train the model
					for epoch in range(__a   ):
										model.train()
										with LocalSGD(
										    accelerator=__a ,					model=__a ,					local_sgd_steps=__a ,					enabled=local_sgd_steps is not None   ) as local_sgd:
															for step, batch in enumerate(__a   ):
																				# We could avoid this line since we set the accelerator with `device_placement=True`.
																				batch.to(accelerator.device   )
																				# New code #
																				# We use the new `accumulate` context manager to perform gradient accumulation
																				# We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
																				with accelerator.accumulate(__a   ):
																									UpperCamelCase__   :Dict												= model(**__a   )
																									UpperCamelCase__   :Optional[Any]												= output.loss
																									accelerator.backward(__a   )
																									optimizer.step()
																									lr_scheduler.step()
																									optimizer.zero_grad()
																									# LocalSGD-specific line
																									local_sgd.step()

										model.eval()
										for step, batch in enumerate(__a   ):
															# We could avoid this line since we set the accelerator with `device_placement=True`.
															batch.to(accelerator.device   )
															with torch.no_grad():
																				UpperCamelCase__   :int												= model(**__a   )
															UpperCamelCase__   :List[str]												= outputs.logits.argmax(dim=-1   )
															UpperCamelCase__					,				UpperCamelCase__   :List[str]												= accelerator.gather_for_metrics((predictions, batch['''labels'''])   )
															metric.add_batch(
															    predictions=__a ,					references=__a ,					)

										UpperCamelCase__   :Optional[int]												= metric.compute()
										# Use accelerator.print to print only on the main process.
										accelerator.print(f'''epoch {epoch}:''' ,					__a   )


def 			a      (			)  ->	Any:



					'''simple docstring'''






					UpperCamelCase__   :int												= argparse.ArgumentParser(description='''Simple example of training script.'''   )
					parser.add_argument(
					    '''--mixed_precision''' ,					type=__a ,					default=__a ,					choices=['''no''', '''fp16''', '''bf16''', '''fp8'''] ,					help='''Whether to use mixed precision. Choose'''
					    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
					    '''and an Nvidia Ampere GPU.''' ,					)
					# New Code #
					parser.add_argument(
					    '''--gradient_accumulation_steps''' ,					type=__a ,					default=1 ,					help='''The number of minibatches to be ran before gradients are accumulated.''' ,					)
					parser.add_argument(
					    '''--local_sgd_steps''' ,					type=__a ,					default=8 ,					help='''Number of local SGD steps or None to disable local SGD'''   )
					parser.add_argument('''--cpu''' ,					action='''store_true''' ,					help='''If passed, will train on the CPU.'''   )
					UpperCamelCase__   :List[Any]												= parser.parse_args()
					UpperCamelCase__   :Optional[Any]												= {'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
					training_function(__a ,					__a   )


if __name__ == "__main__":
		main()



'''simple docstring'''
import unittest
from dataclasses import dataclass

import pytest

from accelerate.commands.config.config_args import SageMakerConfig
from accelerate.utils import ComputeEnvironment
from accelerate.utils.launch import _convert_nargs_to_dict
@dataclass
class    lowercase	(							A__					):

		"""simple docstring"""

		_a     =				ComputeEnvironment.AMAZON_SAGEMAKER
		_a     =				True
		_a     =				'ml.p3.2xlarge'
		_a     =				'accelerate_sagemaker_execution_role'
		_a     =				'hf-sm'
		_a     =				'us-east-1'
		_a     =				1
		_a     =				'accelerate-sagemaker-1'
		_a     =				'1.6'
		_a     =				'4.4'
		_a     =				'train.py'
		_a     =				[
		    '--model_name_or_path',
		    'bert',
		    '--do_train',
		    'False',
		    '--epochs',
		    '3',
		    '--learning_rate',
		    '5e-5',
		    '--max_steps',
		    '50.5',
		]
		_a     =				[
		    '--model_name_or_path',
		    'bert',
		    '--do_train',
		    '--do_test',
		    'False',
		    '--do_predict',
		    '--epochs',
		    '3',
		    '--learning_rate',
		    '5e-5',
		    '--max_steps',
		    '50.5',
		]



class    lowercase	(							unittest.TestCase					):

		"""simple docstring"""

		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :Union[str, Any]												= _convert_nargs_to_dict(MockLaunchConfig.success_training_script_args							)
							assert isinstance(converted_args['''model_name_or_path''']    ,   UpperCamelCase_							)
							assert isinstance(converted_args['''do_train''']    ,   UpperCamelCase_							)
							assert isinstance(converted_args['''epochs''']    ,   UpperCamelCase_							)
							assert isinstance(converted_args['''learning_rate''']    ,   UpperCamelCase_							)
							assert isinstance(converted_args['''max_steps''']    ,   UpperCamelCase_							)

							with pytest.raises(UpperCamelCase_							):
												_convert_nargs_to_dict(MockLaunchConfig.fail_training_script_args							)