



"""simple docstring"""






# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from packaging import version

from .. import __version__
from .constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD
from .doc import (
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    copy_func,
    replace_return_docstrings,
)
from .generic import (
    ContextManagers,
    ExplicitEnum,
    ModelOutput,
    PaddingStrategy,
    TensorType,
    add_model_info_to_auto_map,
    cached_property,
    can_return_loss,
    expand_dims,
    find_labels,
    flatten_dict,
    infer_framework,
    is_jax_tensor,
    is_numpy_array,
    is_tensor,
    is_tf_symbolic_tensor,
    is_tf_tensor,
    is_torch_device,
    is_torch_dtype,
    is_torch_tensor,
    reshape,
    squeeze,
    strtobool,
    tensor_size,
    to_numpy,
    to_py_obj,
    transpose,
    working_or_temp_dir,
)
from .hub import (
    CLOUDFRONT_DISTRIB_PREFIX,
    DISABLE_TELEMETRY,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_PREFIX,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    S3_BUCKET_PREFIX,
    TRANSFORMERS_CACHE,
    TRANSFORMERS_DYNAMIC_MODULE_NAME,
    EntryNotFoundError,
    PushToHubMixin,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    cached_file,
    default_cache_path,
    define_sagemaker_information,
    download_url,
    extract_commit_hash,
    get_cached_models,
    get_file_from_repo,
    get_full_repo_name,
    has_file,
    http_user_agent,
    is_offline_mode,
    is_remote_url,
    move_cache,
    send_example_telemetry,
    try_to_load_from_cache,
)
from .import_utils import (
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    TORCH_FX_REQUIRED_VERSION,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    DummyObject,
    OptionalDependencyNotAvailable,
    _LazyModule,
    ccl_version,
    direct_transformers_import,
    get_torch_version,
    is_accelerate_available,
    is_apex_available,
    is_bitsandbytes_available,
    is_bsa_available,
    is_coloredlogs_available,
    is_cython_available,
    is_datasets_available,
    is_decord_available,
    is_detectrona_available,
    is_faiss_available,
    is_flax_available,
    is_ftfy_available,
    is_in_notebook,
    is_ipex_available,
    is_jieba_available,
    is_jumanpp_available,
    is_kenlm_available,
    is_keras_nlp_available,
    is_librosa_available,
    is_natten_available,
    is_ninja_available,
    is_onnx_available,
    is_openai_available,
    is_optimum_available,
    is_pandas_available,
    is_peft_available,
    is_phonemizer_available,
    is_protobuf_available,
    is_psutil_available,
    is_pyanvml_available,
    is_pyctcdecode_available,
    is_pytesseract_available,
    is_pytest_available,
    is_pytorch_quantization_available,
    is_rjieba_available,
    is_sacremoses_available,
    is_safetensors_available,
    is_sagemaker_dp_enabled,
    is_sagemaker_mp_enabled,
    is_scipy_available,
    is_sentencepiece_available,
    is_seqio_available,
    is_sklearn_available,
    is_soundfile_availble,
    is_spacy_available,
    is_speech_available,
    is_sudachi_available,
    is_tensorflow_probability_available,
    is_tensorflow_text_available,
    is_tfaonnx_available,
    is_tf_available,
    is_timm_available,
    is_tokenizers_available,
    is_torch_available,
    is_torch_bfaa_available,
    is_torch_bfaa_cpu_available,
    is_torch_bfaa_gpu_available,
    is_torch_compile_available,
    is_torch_cuda_available,
    is_torch_fx_available,
    is_torch_fx_proxy,
    is_torch_mps_available,
    is_torch_neuroncore_available,
    is_torch_tensorrt_fx_available,
    is_torch_tfaa_available,
    is_torch_tpu_available,
    is_torchaudio_available,
    is_torchdistx_available,
    is_torchdynamo_available,
    is_torchvision_available,
    is_training_run_on_sagemaker,
    is_vision_available,
    requires_backends,
    torch_only_method,
)


lowerCAmelCase__              =							'''pytorch_model.bin'''
lowerCAmelCase__              =							'''pytorch_model.bin.index.json'''
lowerCAmelCase__              =							'''adapter_config.json'''
lowerCAmelCase__              =							'''adapter_model.bin'''
lowerCAmelCase__              =							'''adapter_model.safetensors'''
lowerCAmelCase__              =							'''tf_model.h5'''
lowerCAmelCase__              =							'''tf_model.h5.index.json'''
lowerCAmelCase__              =							'''model.ckpt'''
lowerCAmelCase__              =							'''flax_model.msgpack'''
lowerCAmelCase__              =							'''flax_model.msgpack.index.json'''
lowerCAmelCase__              =							'''model.safetensors'''
lowerCAmelCase__              =							'''model.safetensors.index.json'''
lowerCAmelCase__              =							'''config.json'''
lowerCAmelCase__              =							'''preprocessor_config.json'''
lowerCAmelCase__              =							FEATURE_EXTRACTOR_NAME
lowerCAmelCase__              =							'''generation_config.json'''
lowerCAmelCase__              =							'''modelcard.json'''

lowerCAmelCase__              =							'''▁'''
lowerCAmelCase__              =							SENTENCEPIECE_UNDERLINE  # Kept for backward compatibility

lowerCAmelCase__              =							[
    [[0, 1, 0, 1], [1, 0, 0, 1]]
] * 2  # Needs to have 0s and 1s only since XLM uses it for langs too.
lowerCAmelCase__              =							[[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]
lowerCAmelCase__              =							[[1, 1, 1, 1, 1], [1, 1, 1, 0, 0], [0, 0, 0, 1, 1]]






def 			snake_case_    ( A_   :						Dict		):


       '''simple docstring'''





       if version.parse(A_		) < version.parse(A_		):
              if "dev" in min_version:
                     _lowerCamelCase		:							Optional[Any]            =					(
                         '''This example requires a source install from HuggingFace Transformers (see '''
                         '''`https://huggingface.co/docs/transformers/installation#install-from-source`),'''
                     )
              else:
                     _lowerCamelCase		:							Union[str, Any]            =					F'''This example requires a minimum version of {min_version},'''
              error_message += F''' but the version found is {__version__}.\n'''
              raise ImportError(
                  error_message
                  + '''Check out https://github.com/huggingface/transformers/tree/main/examples#important-note for the examples corresponding to other '''
                  '''versions of HuggingFace Transformers.'''		)








'''simple docstring'''

import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.local_sgd import LocalSGD


########################################################################
# This is a fully working simple example to use Accelerate
# with LocalSGD, which is a method to synchronize model
# parameters every K batches. It is different, but complementary
# to gradient accumulation.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


a    :							str										=    16
a    :							Union[str, Any]										=    32


def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase = 16      )						->   Union[str, Any]:

					'''simple docstring'''

					snake_case_		     =							AutoTokenizer.from_pretrained('''bert-base-cased'''      )
					snake_case_		     =							load_dataset('''glue''', '''mrpc'''      )

					def tokenize_function(__UpperCAmelCase      ):
										# max_length=None => use the model max length (it's actually the default)
										snake_case_		     =							tokenizer(examples['''sentence1'''], examples['''sentence2'''], truncation=__UpperCAmelCase, max_length=__UpperCAmelCase      )
										return outputs

					# Apply the method we just defined to all the examples in all the splits of the dataset
					# starting with the main process first:
					with accelerator.main_process_first():
										snake_case_		     =							datasets.map(
										    __UpperCAmelCase, batched=__UpperCAmelCase, remove_columns=['''idx''', '''sentence1''', '''sentence2'''], )

					# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
					# transformers library
					snake_case_		     =							tokenized_datasets.rename_column('''label''', '''labels'''      )

					def collate_fn(__UpperCAmelCase      ):
										# On TPU it's best to pad everything to the same length or training will be very slow.
										snake_case_		     =							128 if accelerator.distributed_type == DistributedType.TPU else None
										# When using mixed precision we want round multiples of 8/16
										if accelerator.mixed_precision == "fp8":
															snake_case_		     =							16
										elif accelerator.mixed_precision != "no":
															snake_case_		     =							8
										else:
															snake_case_		     =							None

										return tokenizer.pad(
										    __UpperCAmelCase, padding='''longest''', max_length=__UpperCAmelCase, pad_to_multiple_of=__UpperCAmelCase, return_tensors='''pt''', )

					# Instantiate dataloaders.
					snake_case_		     =							DataLoader(
					    tokenized_datasets['''train'''], shuffle=__UpperCAmelCase, collate_fn=__UpperCAmelCase, batch_size=__UpperCAmelCase      )
					snake_case_		     =							DataLoader(
					    tokenized_datasets['''validation'''], shuffle=__UpperCAmelCase, collate_fn=__UpperCAmelCase, batch_size=__UpperCAmelCase      )

					return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
							from accelerate.test_utils.training import mocked_dataloaders

							a    :							Optional[Any]										=    mocked_dataloaders  # noqa: F811


def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase      )						->   Optional[Any]:

					'''simple docstring'''

					if os.environ.get('''TESTING_MOCKED_DATALOADERS''', __UpperCAmelCase      ) == "1":
										snake_case_		     =							2
					# New Code #
					snake_case_		     =							int(args.gradient_accumulation_steps      )
					snake_case_		     =							int(args.local_sgd_steps      )
					# Initialize accelerator
					snake_case_		     =							Accelerator(
					    cpu=args.cpu, mixed_precision=args.mixed_precision, gradient_accumulation_steps=__UpperCAmelCase      )
					if accelerator.distributed_type not in [DistributedType.NO, DistributedType.MULTI_CPU, DistributedType.MULTI_GPU]:
										raise NotImplementedError('''LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'''      )
					# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
					snake_case_		     =							config['''lr''']
					snake_case_		     =							int(config['''num_epochs''']      )
					snake_case_		     =							int(config['''seed''']      )
					snake_case_		     =							int(config['''batch_size''']      )

					snake_case_		     =							evaluate.load('''glue''', '''mrpc'''      )

					set_seed(__UpperCAmelCase      )
					snake_case_     ,snake_case_		     =							get_dataloaders(__UpperCAmelCase, __UpperCAmelCase      )
					# Instantiate the model (we build the model here so that the seed also control new weights initialization)
					snake_case_		     =							AutoModelForSequenceClassification.from_pretrained('''bert-base-cased''', return_dict=__UpperCAmelCase      )

					# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
					# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
					# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
					snake_case_		     =							model.to(accelerator.device      )

					# Instantiate optimizer
					snake_case_		     =							AdamW(params=model.parameters(), lr=__UpperCAmelCase      )

					# Instantiate scheduler
					snake_case_		     =							get_linear_schedule_with_warmup(
					    optimizer=__UpperCAmelCase, num_warmup_steps=100, num_training_steps=(len(__UpperCAmelCase      ) * num_epochs), )

					# Prepare everything
					# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
					# prepare method.
					snake_case_     ,snake_case_     ,snake_case_     ,snake_case_     ,snake_case_		     =							accelerator.prepare(
					    __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase      )

					# Now we train the model
					for epoch in range(__UpperCAmelCase      ):
										model.train()
										with LocalSGD(
										    accelerator=__UpperCAmelCase, model=__UpperCAmelCase, local_sgd_steps=__UpperCAmelCase, enabled=local_sgd_steps is not None      ) as local_sgd:
															for step, batch in enumerate(__UpperCAmelCase      ):
																				# We could avoid this line since we set the accelerator with `device_placement=True`.
																				batch.to(accelerator.device      )
																				# New code #
																				# We use the new `accumulate` context manager to perform gradient accumulation
																				# We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
																				with accelerator.accumulate(__UpperCAmelCase      ):
																									snake_case_		     =							model(**__UpperCAmelCase      )
																									snake_case_		     =							output.loss
																									accelerator.backward(__UpperCAmelCase      )
																									optimizer.step()
																									lr_scheduler.step()
																									optimizer.zero_grad()
																									# LocalSGD-specific line
																									local_sgd.step()

										model.eval()
										for step, batch in enumerate(__UpperCAmelCase      ):
															# We could avoid this line since we set the accelerator with `device_placement=True`.
															batch.to(accelerator.device      )
															with torch.no_grad():
																				snake_case_		     =							model(**__UpperCAmelCase      )
															snake_case_		     =							outputs.logits.argmax(dim=-1      )
															snake_case_     ,snake_case_		     =							accelerator.gather_for_metrics((predictions, batch['''labels'''])      )
															metric.add_batch(
															    predictions=__UpperCAmelCase, references=__UpperCAmelCase, )

										snake_case_		     =							metric.compute()
										# Use accelerator.print to print only on the main process.
										accelerator.print(F"epoch {epoch}:", __UpperCAmelCase      )


def 						__magic_name__       (      )						->   str:

					'''simple docstring'''

					snake_case_		     =							argparse.ArgumentParser(description='''Simple example of training script.'''      )
					parser.add_argument(
					    '''--mixed_precision''', type=__UpperCAmelCase, default=__UpperCAmelCase, choices=['''no''', '''fp16''', '''bf16''', '''fp8'''], help='''Whether to use mixed precision. Choose'''
					    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
					    '''and an Nvidia Ampere GPU.''', )
					# New Code #
					parser.add_argument(
					    '''--gradient_accumulation_steps''', type=__UpperCAmelCase, default=1, help='''The number of minibatches to be ran before gradients are accumulated.''', )
					parser.add_argument(
					    '''--local_sgd_steps''', type=__UpperCAmelCase, default=8, help='''Number of local SGD steps or None to disable local SGD'''      )
					parser.add_argument('''--cpu''', action='''store_true''', help='''If passed, will train on the CPU.'''      )
					snake_case_		     =							parser.parse_args()
					snake_case_		     =							{'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
					training_function(__UpperCAmelCase, __UpperCAmelCase      )


if __name__ == "__main__":
							main()

