




"""simple docstring"""


import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate
# and perform gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


A_						=				16
A_						=				32


def 					UpperCAmelCase__       (snake_case__							:					Accelerator		,   snake_case__							:					int = 16    ):
  """simple docstring"""

  _snake_case							:		Union[str, Any]								=  AutoTokenizer.from_pretrained("""bert-base-cased"""    )
  _snake_case							:		Tuple								=  load_dataset("""glue"""		,   """mrpc"""    )

  def tokenize_function(snake_case__							:					List[str]    ):
    # max_length=None => use the model max length (it's actually the default)
    _snake_case							:		str								=  tokenizer(examples["""sentence1"""]		,   examples["""sentence2"""]		,   truncation=_UpperCAmelCase		,   max_length=_UpperCAmelCase    )
    return outputs

  # Apply the method we just defined to all the examples in all the splits of the dataset
  # starting with the main process first:
  with accelerator.main_process_first():
    _snake_case							:		str								=  datasets.map(
        _UpperCAmelCase		,   batched=_UpperCAmelCase		,   remove_columns=["""idx""", """sentence1""", """sentence2"""]		,   )

  # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  # transformers library
  _snake_case							:		Any								=  tokenized_datasets.rename_column("""label"""		,   """labels"""    )

  def collate_fn(snake_case__							:					List[Any]    ):
    # On TPU it's best to pad everything to the same length or training will be very slow.
    _snake_case							:		Optional[Any]								=  1_28 if accelerator.distributed_type == DistributedType.TPU else None
    # When using mixed precision we want round multiples of 8/16
    if accelerator.mixed_precision == "fp8":
      _snake_case							:		Union[str, Any]								=  16
    elif accelerator.mixed_precision != "no":
      _snake_case							:		Optional[Any]								=  8
    else:
      _snake_case							:		Union[str, Any]								=  None

    return tokenizer.pad(
        _UpperCAmelCase		,   padding="""longest"""		,   max_length=_UpperCAmelCase		,   pad_to_multiple_of=_UpperCAmelCase		,   return_tensors="""pt"""		,   )

  # Instantiate dataloaders.
  _snake_case							:		str								=  DataLoader(
      tokenized_datasets["""train"""]		,   shuffle=_UpperCAmelCase		,   collate_fn=_UpperCAmelCase		,   batch_size=_UpperCAmelCase    )
  _snake_case							:		List[Any]								=  DataLoader(
      tokenized_datasets["""validation"""]		,   shuffle=_UpperCAmelCase		,   collate_fn=_UpperCAmelCase		,   batch_size=_UpperCAmelCase    )

  return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
     from accelerate.test_utils.training import mocked_dataloaders

     A_						=				mocked_dataloaders  # noqa: F811


def 					UpperCAmelCase__       (snake_case__							:					Optional[int]		,   snake_case__							:					Any    ):
  """simple docstring"""

  if os.environ.get("""TESTING_MOCKED_DATALOADERS"""		,   _UpperCAmelCase    ) == "1":
    _snake_case							:		int								=  2
  # New Code #
  _snake_case							:		int								=  int(args.gradient_accumulation_steps    )
  # Initialize accelerator
  _snake_case							:		Optional[Any]								=  Accelerator(
      cpu=args.cpu		,   mixed_precision=args.mixed_precision		,   gradient_accumulation_steps=_UpperCAmelCase    )
  if accelerator.distributed_type == DistributedType.TPU and gradient_accumulation_steps > 1:
    raise NotImplementedError(
        """Gradient accumulation on TPUs is currently not supported. Pass `gradient_accumulation_steps=1`"""    )
  # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  _snake_case							:		List[Any]								=  config["""lr"""]
  _snake_case							:		List[str]								=  int(config["""num_epochs"""]    )
  _snake_case							:		Any								=  int(config["""seed"""]    )
  _snake_case							:		Tuple								=  int(config["""batch_size"""]    )

  _snake_case							:		Any								=  evaluate.load("""glue"""		,   """mrpc"""    )

  set_seed(_UpperCAmelCase    )
  _snake_case							:		Tuple								=  get_dataloaders(_UpperCAmelCase		,   _UpperCAmelCase    )
  # Instantiate the model (we build the model here so that the seed also control new weights initialization)
  _snake_case							:		List[str]								=  AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""		,   return_dict=_UpperCAmelCase    )

  # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
  # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
  # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
  _snake_case							:		Tuple								=  model.to(accelerator.device    )

  # Instantiate optimizer
  _snake_case							:		Dict								=  AdamW(params=model.parameters()		,   lr=_UpperCAmelCase    )

  # Instantiate scheduler
  _snake_case							:		Any								=  get_linear_schedule_with_warmup(
      optimizer=_UpperCAmelCase		,   num_warmup_steps=1_00		,   num_training_steps=(len(_UpperCAmelCase    ) * num_epochs)		,   )

  # Prepare everything
  # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
  # prepare method.
  _snake_case							:		Optional[Any]								=  accelerator.prepare(
      _UpperCAmelCase		,   _UpperCAmelCase		,   _UpperCAmelCase		,   _UpperCAmelCase		,   _UpperCAmelCase    )

  # Now we train the model
  for epoch in range(_UpperCAmelCase    ):
    model.train()
    for step, batch in enumerate(_UpperCAmelCase    ):
      # We could avoid this line since we set the accelerator with `device_placement=True`.
      batch.to(accelerator.device    )
      # New code #
      # We use the new `accumulate` context manager to perform gradient accumulation
      # We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
      with accelerator.accumulate(_UpperCAmelCase    ):
        _snake_case							:		int								=  model(**_UpperCAmelCase    )
        _snake_case							:		Union[str, Any]								=  output.loss
        accelerator.backward(_UpperCAmelCase    )
        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()

    model.eval()
    for step, batch in enumerate(_UpperCAmelCase    ):
      # We could avoid this line since we set the accelerator with `device_placement=True`.
      batch.to(accelerator.device    )
      with torch.no_grad():
        _snake_case							:		Optional[Any]								=  model(**_UpperCAmelCase    )
      _snake_case							:		Any								=  outputs.logits.argmax(dim=-1    )
      _snake_case							:		Dict								=  accelerator.gather_for_metrics((predictions, batch["""labels"""])    )
      metric.add_batch(
          predictions=_UpperCAmelCase		,   references=_UpperCAmelCase		,   )

    _snake_case							:		List[str]								=  metric.compute()
    # Use accelerator.print to print only on the main process.
    accelerator.print(F"epoch {epoch}:"		,   _UpperCAmelCase    )


def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		List[Any]								=  argparse.ArgumentParser(description="""Simple example of training script."""    )
  parser.add_argument(
      """--mixed_precision"""		,   type=_UpperCAmelCase		,   default=_UpperCAmelCase		,   choices=["""no""", """fp16""", """bf16""", """fp8"""]		,   help="""Whether to use mixed precision. Choose"""
      """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
      """and an Nvidia Ampere GPU."""		,   )
  # New Code #
  parser.add_argument(
      """--gradient_accumulation_steps"""		,   type=_UpperCAmelCase		,   default=1		,   help="""The number of minibatches to be ran before gradients are accumulated."""		,   )
  parser.add_argument("""--cpu"""		,   action="""store_true"""		,   help="""If passed, will train on the CPU."""    )
  _snake_case							:		int								=  parser.parse_args()
  _snake_case							:		int								=  {"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
  training_function(_UpperCAmelCase		,   _UpperCAmelCase    )


if __name__ == "__main__":
     main()






'''simple docstring'''







from ...configuration_utils import PretrainedConfig
from ...utils import logging


A__:    Dict					    =       logging.get_logger(__name__)

A__:    Tuple					    =       {
    '''weiweishi/roc-bert-base-zh''': '''https://huggingface.co/weiweishi/roc-bert-base-zh/resolve/main/config.json''',
}







class A__       (						UpperCAmelCase__      ):
      __UpperCamelCase       :  Tuple	       =							"roc_bert"






      def __init__( self			:Optional[int]     ,						SCREAMING_SNAKE_CASE			:Tuple=3_0_5_2_2     ,						SCREAMING_SNAKE_CASE			:List[str]=7_6_8     ,						SCREAMING_SNAKE_CASE			:Dict=1_2     ,						SCREAMING_SNAKE_CASE			:List[str]=1_2     ,						SCREAMING_SNAKE_CASE			:Tuple=3_0_7_2     ,						SCREAMING_SNAKE_CASE			:List[Any]="gelu"     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]=0.1     ,						SCREAMING_SNAKE_CASE			:List[Any]=0.1     ,						SCREAMING_SNAKE_CASE			:int=5_1_2     ,						SCREAMING_SNAKE_CASE			:Optional[Any]=2     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]=0.02     ,						SCREAMING_SNAKE_CASE			:Optional[Any]=1e-12     ,						SCREAMING_SNAKE_CASE			:Any=True     ,						SCREAMING_SNAKE_CASE			:List[Any]=0     ,						SCREAMING_SNAKE_CASE			:Optional[int]="absolute"     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]=None     ,						SCREAMING_SNAKE_CASE			:List[Any]=True     ,						SCREAMING_SNAKE_CASE			:int=True     ,						SCREAMING_SNAKE_CASE			:Optional[int]=7_6_8     ,						SCREAMING_SNAKE_CASE			:Optional[Any]=9_1_0     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]=5_1_2     ,						SCREAMING_SNAKE_CASE			:str=2_4_8_5_8     ,						SCREAMING_SNAKE_CASE			:List[Any]=True     ,						**SCREAMING_SNAKE_CASE			:Tuple     ,						)       ->					Optional[int]:




             '''simple docstring'''
             _a    : List[str]     					=vocab_size
             _a    : List[str]     					=max_position_embeddings
             _a    : Optional[Any]     					=hidden_size
             _a    : List[Any]     					=num_hidden_layers
             _a    : List[str]     					=num_attention_heads
             _a    : int     					=intermediate_size
             _a    : Any     					=hidden_act
             _a    : Dict     					=hidden_dropout_prob
             _a    : int     					=attention_probs_dropout_prob
             _a    : str     					=initializer_range
             _a    : Optional[int]     					=type_vocab_size
             _a    : Any     					=layer_norm_eps
             _a    : Any     					=use_cache
             _a    : Optional[int]     					=enable_pronunciation
             _a    : Optional[Any]     					=enable_shape
             _a    : Optional[Any]     					=pronunciation_embed_dim
             _a    : Tuple     					=pronunciation_vocab_size
             _a    : Union[str, Any]     					=shape_embed_dim
             _a    : Any     					=shape_vocab_size
             _a    : Tuple     					=concat_input
             _a    : List[str]     					=position_embedding_type
             _a    : List[str]     					=classifier_dropout
             super().__init__(pad_token_id=SCREAMING_SNAKE_CASE     ,						**SCREAMING_SNAKE_CASE	)


