
'''simple docstring'''



import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to properly calculate the metrics on the
# validation dataset when in a distributed system, and builds off the
# `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


_lowercase          =						16
_lowercase          =						32


def A (__lowerCamelCase :Accelerator					,      __lowerCamelCase :int = 16						):
						_lowerCAmelCase    		=					AutoTokenizer.from_pretrained("""bert-base-cased"""						)
						_lowerCAmelCase    		=					load_dataset("""glue"""					,      """mrpc"""						)

						def tokenize_function(__lowerCamelCase :Tuple						):
												# max_length=None => use the model max length (it's actually the default)
												_lowerCAmelCase    		=					tokenizer(examples["""sentence1"""]					,      examples["""sentence2"""]					,      truncation=_A					,      max_length=_A						)
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						# starting with the main process first:
						with accelerator.main_process_first():
												_lowerCAmelCase    		=					datasets.map(
												    _A					,      batched=_A					,      remove_columns=["""idx""", """sentence1""", """sentence2"""]					,      )

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						_lowerCAmelCase    		=					tokenized_datasets.rename_column("""label"""					,      """labels"""						)

						def collate_fn(__lowerCamelCase :Union[str, Any]						):
												# On TPU it's best to pad everything to the same length or training will be very slow.
												_lowerCAmelCase    		=					128 if accelerator.distributed_type == DistributedType.TPU else None
												# When using mixed precision we want round multiples of 8/16
												if accelerator.mixed_precision == "fp8":
																		_lowerCAmelCase    		=					16
												elif accelerator.mixed_precision != "no":
																		_lowerCAmelCase    		=					8
												else:
																		_lowerCAmelCase    		=					None

												return tokenizer.pad(
												    _A					,      padding="""longest"""					,      max_length=_A					,      pad_to_multiple_of=_A					,      return_tensors="""pt"""					,      )

						# Instantiate dataloaders.
						_lowerCAmelCase    		=					DataLoader(
						    tokenized_datasets["""train"""]					,      shuffle=_A					,      collate_fn=_A					,      batch_size=_A						)
						_lowerCAmelCase    		=					DataLoader(
						    tokenized_datasets["""validation"""]					,      shuffle=_A					,      collate_fn=_A					,      batch_size=_A						)

						return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
						from accelerate.test_utils.training import mocked_dataloaders

						_lowercase          =						mocked_dataloaders  # noqa: F811


def A (__lowerCamelCase :List[Any]					,      __lowerCamelCase :Optional[Any]						):
						# For testing only
						if os.environ.get("""TESTING_MOCKED_DATALOADERS"""					,      _A						) == "1":
												_lowerCAmelCase    		=					2
						# Initialize accelerator
						_lowerCAmelCase    		=					Accelerator(cpu=args.cpu					,      mixed_precision=args.mixed_precision						)
						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						_lowerCAmelCase    		=					config['''lr''']
						_lowerCAmelCase    		=					int(config["""num_epochs"""]						)
						_lowerCAmelCase    		=					int(config["""seed"""]						)
						_lowerCAmelCase    		=					int(config["""batch_size"""]						)

						_lowerCAmelCase    		=					evaluate.load("""glue"""					,      """mrpc"""						)

						# If the batch size is too big we use gradient accumulation
						_lowerCAmelCase    		=					1
						if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
												_lowerCAmelCase    		=					batch_size // MAX_GPU_BATCH_SIZE
												_lowerCAmelCase    		=					MAX_GPU_BATCH_SIZE

						set_seed(_A						)
						_lowerCAmelCase    		=					get_dataloaders(_A					,      _A						)
						# Instantiate the model (we build the model here so that the seed also control new weights initialization)
						_lowerCAmelCase    		=					AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""					,      return_dict=_A						)

						# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
						# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
						# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
						_lowerCAmelCase    		=					model.to(accelerator.device						)

						# Instantiate optimizer
						_lowerCAmelCase    		=					AdamW(params=model.parameters()					,      lr=_A						)

						# Instantiate scheduler
						_lowerCAmelCase    		=					get_linear_schedule_with_warmup(
						    optimizer=_A					,      num_warmup_steps=100					,      num_training_steps=(len(_A						) * num_epochs) // gradient_accumulation_steps					,      )

						# Prepare everything
						# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
						# prepare method.
						_lowerCAmelCase    		=					accelerator.prepare(
						    _A					,      _A					,      _A					,      _A					,      _A						)

						# Now we train the model
						for epoch in range(_A						):
												model.train()
												for step, batch in enumerate(_A						):
																		# We could avoid this line since we set the accelerator with `device_placement=True`.
																		batch.to(accelerator.device						)
																		_lowerCAmelCase    		=					model(**_A						)
																		_lowerCAmelCase    		=					outputs.loss
																		_lowerCAmelCase    		=					loss / gradient_accumulation_steps
																		accelerator.backward(_A						)
																		if step % gradient_accumulation_steps == 0:
																								optimizer.step()
																								lr_scheduler.step()
																								optimizer.zero_grad()

												model.eval()
												_lowerCAmelCase    		=					0
												for step, batch in enumerate(_A						):
																		# We could avoid this line since we set the accelerator with `device_placement=True`.
																		batch.to(accelerator.device						)
																		with torch.no_grad():
																								_lowerCAmelCase    		=					model(**_A						)
																		_lowerCAmelCase    		=					outputs.logits.argmax(dim=-1						)
																		_lowerCAmelCase    		=					accelerator.gather((predictions, batch["""labels"""])						)
																		# New Code #
																		# First we check if it's a distributed system
																		if accelerator.use_distributed:
																								# Then see if we're on the last batch of our eval dataloader
																								if step == len(_A						) - 1:
																														# Last batch needs to be truncated on distributed systems as it contains additional samples
																														_lowerCAmelCase    		=					predictions[: len(eval_dataloader.dataset						) - samples_seen]
																														_lowerCAmelCase    		=					references[: len(eval_dataloader.dataset						) - samples_seen]
																								else:
																														# Otherwise we add the number of samples seen
																														samples_seen += references.shape[0]
																								# All of this can be avoided if you use `Accelerator.gather_for_metrics` instead of `Accelerator.gather`:
																								# accelerator.gather_for_metrics((predictions, batch["labels"]))
																		metric.add_batch(
																		    predictions=_A					,      references=_A					,      )

												_lowerCAmelCase    		=					metric.compute()
												# Use accelerator.print to print only on the main process.
												accelerator.print(f'epoch {epoch}:'					,      _A						)


def A ():
						_lowerCAmelCase    		=					argparse.ArgumentParser(description="""Simple example of training script."""						)
						parser.add_argument(
						    """--mixed_precision"""					,      type=_A					,      default=_A					,      choices=["""no""", """fp16""", """bf16""", """fp8"""]					,      help="""Whether to use mixed precision. Choose"""
						    """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
						    """and an Nvidia Ampere GPU."""					,      )
						parser.add_argument("""--cpu"""					,      action="""store_true"""					,      help="""If passed, will train on the CPU."""						)
						_lowerCAmelCase    		=					parser.parse_args()
						_lowerCAmelCase    		=					{'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
						training_function(_A					,      _A						)


if __name__ == "__main__":
						main()

'''simple docstring'''

import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
UpperCamelCase__  :						Dict					=      logging.getLogger(__name__)







def __UpperCamelCase( ):
   '''simple docstring'''
   UpperCAmelCase__				:					Optional[Any] 					=	argparse.ArgumentParser(
       description='''Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'''     )
   parser.add_argument('''--file_path'''	,      type=_A	,      default='''data/dump.txt'''	,      help='''The path to the data.'''     )
   parser.add_argument('''--tokenizer_type'''	,      type=_A	,      default='''bert'''	,      choices=['''bert''', '''roberta''', '''gpt2''']     )
   parser.add_argument('''--tokenizer_name'''	,      type=_A	,      default='''bert-base-uncased'''	,      help='''The tokenizer to use.'''     )
   parser.add_argument('''--dump_file'''	,      type=_A	,      default='''data/dump'''	,      help='''The dump file prefix.'''     )
   UpperCAmelCase__				:					Optional[int] 					=	parser.parse_args()

   logger.info(F'''Loading Tokenizer ({args.tokenizer_name})'''     )
   if args.tokenizer_type == "bert":
      UpperCAmelCase__				:					List[str] 					=	BertTokenizer.from_pretrained(args.tokenizer_name     )
      UpperCAmelCase__				:					Optional[Any] 					=	tokenizer.special_tokens_map['''cls_token''']  # `[CLS]`
      UpperCAmelCase__				:					Union[str, Any] 					=	tokenizer.special_tokens_map['''sep_token''']  # `[SEP]`
   elif args.tokenizer_type == "roberta":
      UpperCAmelCase__				:					Union[str, Any] 					=	RobertaTokenizer.from_pretrained(args.tokenizer_name     )
      UpperCAmelCase__				:					List[Any] 					=	tokenizer.special_tokens_map['''cls_token''']  # `<s>`
      UpperCAmelCase__				:					Dict 					=	tokenizer.special_tokens_map['''sep_token''']  # `</s>`
   elif args.tokenizer_type == "gpt2":
      UpperCAmelCase__				:					List[str] 					=	GPTaTokenizer.from_pretrained(args.tokenizer_name     )
      UpperCAmelCase__				:					List[Any] 					=	tokenizer.special_tokens_map['''bos_token''']  # `<|endoftext|>`
      UpperCAmelCase__				:					Optional[Any] 					=	tokenizer.special_tokens_map['''eos_token''']  # `<|endoftext|>`

   logger.info(F'''Loading text from {args.file_path}'''     )
   with open(args.file_path	,      '''r'''	,      encoding='''utf8'''     ) as fp:
      UpperCAmelCase__				:					List[Any] 					=	fp.readlines()

   logger.info('''Start encoding'''     )
   logger.info(F'''{len(_A     )} examples to process.'''     )

   UpperCAmelCase__				:					List[str] 					=	[]
   UpperCAmelCase__				:					Union[str, Any] 					=	0
   UpperCAmelCase__				:					Optional[int] 					=	1_00_00
   UpperCAmelCase__				:					Tuple 					=	time.time()
   for text in data:
      UpperCAmelCase__				:					Any 					=	F'''{bos} {text.strip()} {sep}'''
      UpperCAmelCase__				:					int 					=	tokenizer.encode(_A	,      add_special_tokens=_A     )
      rslt.append(_A     )

      iter += 1
      if iter % interval == 0:
         UpperCAmelCase__				:					int 					=	time.time()
         logger.info(F'''{iter} examples processed. - {(end-start):.2f}s/{interval}expl'''     )
         UpperCAmelCase__				:					Optional[Any] 					=	time.time()
   logger.info('''Finished binarization'''     )
   logger.info(F'''{len(_A     )} examples processed.'''     )

   UpperCAmelCase__				:					Dict 					=	F'''{args.dump_file}.{args.tokenizer_name}.pickle'''
   UpperCAmelCase__				:					Dict 					=	tokenizer.vocab_size
   if vocab_size < (1 << 16):
      UpperCAmelCase__				:					Any 					=	[np.uintaa(_A     ) for d in rslt]
   else:
      UpperCAmelCase__				:					str 					=	[np.intaa(_A     ) for d in rslt]
   random.shuffle(rslt_     )
   logger.info(F'''Dump to {dp_file}'''     )
   with open(_A	,      '''wb'''     ) as handle:
      pickle.dump(rslt_	,      _A	,      protocol=pickle.HIGHEST_PROTOCOL     )


if __name__ == "__main__":
  main()




