


import argparse

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


lowerCAmelCase       =							16
lowerCAmelCase       =							32
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE = 16     ):


		"""simple docstring"""




		lowercase__            =      AutoTokenizer.from_pretrained('''bert-base-cased'''     )
		lowercase__            =      load_dataset('''glue'''						,				'''mrpc'''     )

		def tokenize_function(SCREAMING_SNAKE_CASE     ):
				# max_length=None => use the model max length (it's actually the default)
				lowercase__            =      tokenizer(examples['''sentence1''']						,				examples['''sentence2''']						,				truncation=SCREAMING_SNAKE_CASE						,				max_length=SCREAMING_SNAKE_CASE     )
				return outputs

		# Apply the method we just defined to all the examples in all the splits of the dataset
		# starting with the main process first:
		with accelerator.main_process_first():
				lowercase__            =      datasets.map(
				    SCREAMING_SNAKE_CASE						,				batched=SCREAMING_SNAKE_CASE						,				remove_columns=['''idx''', '''sentence1''', '''sentence2''']						,				)

		# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
		# transformers library
		lowercase__            =      tokenized_datasets.rename_column('''label'''						,				'''labels'''     )

		def collate_fn(SCREAMING_SNAKE_CASE     ):
				# On TPU it's best to pad everything to the same length or training will be very slow.
				lowercase__            =      1_28 if accelerator.distributed_type == DistributedType.TPU else None
				# When using mixed precision we want round multiples of 8/16
				if accelerator.mixed_precision == "fp8":
						lowercase__            =      16
				elif accelerator.mixed_precision != "no":
						lowercase__            =      8
				else:
						lowercase__            =      None

				return tokenizer.pad(
				    SCREAMING_SNAKE_CASE						,				padding='''longest'''						,				max_length=SCREAMING_SNAKE_CASE						,				pad_to_multiple_of=SCREAMING_SNAKE_CASE						,				return_tensors='''pt'''						,				)

		# Instantiate dataloaders.
		lowercase__            =      DataLoader(
		    tokenized_datasets['''train''']						,				shuffle=SCREAMING_SNAKE_CASE						,				collate_fn=SCREAMING_SNAKE_CASE						,				batch_size=SCREAMING_SNAKE_CASE						,				drop_last=SCREAMING_SNAKE_CASE     )
		lowercase__            =      DataLoader(
		    tokenized_datasets['''validation''']						,				shuffle=SCREAMING_SNAKE_CASE						,				collate_fn=SCREAMING_SNAKE_CASE						,				batch_size=SCREAMING_SNAKE_CASE						,				drop_last=(accelerator.mixed_precision == '''fp8''')						,				)

		return train_dataloader, eval_dataloader
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		lowercase__            =      Accelerator(cpu=args.cpu						,				mixed_precision=args.mixed_precision     )
		# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
		lowercase__            =      config['''lr''']
		lowercase__            =      int(config['''num_epochs''']     )
		lowercase__            =      int(config['''seed''']     )
		lowercase__            =      int(config['''batch_size''']     )

		lowercase__            =      evaluate.load('''glue'''						,				'''mrpc'''     )

		# If the batch size is too big we use gradient accumulation
		lowercase__            =      1
		if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
				lowercase__            =      batch_size // MAX_GPU_BATCH_SIZE
				lowercase__            =      MAX_GPU_BATCH_SIZE

		set_seed(SCREAMING_SNAKE_CASE     )
		lowercase__     ,		lowercase__            =      get_dataloaders(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     )
		# Instantiate the model (we build the model here so that the seed also control new weights initialization)
		lowercase__            =      AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''						,				return_dict=SCREAMING_SNAKE_CASE     )

		# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
		# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
		# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
		lowercase__            =      model.to(accelerator.device     )
		# Instantiate optimizer
		lowercase__            =      AdamW(params=model.parameters()						,				lr=SCREAMING_SNAKE_CASE     )

		# Instantiate scheduler
		lowercase__            =      get_linear_schedule_with_warmup(
		    optimizer=SCREAMING_SNAKE_CASE						,				num_warmup_steps=1_00						,				num_training_steps=(len(SCREAMING_SNAKE_CASE     ) * num_epochs) // gradient_accumulation_steps						,				)

		# Prepare everything
		# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
		# prepare method.

		lowercase__     ,		lowercase__     ,		lowercase__     ,		lowercase__     ,		lowercase__            =      accelerator.prepare(
		    SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     )

		# Now we train the model
		for epoch in range(SCREAMING_SNAKE_CASE     ):
				model.train()
				for step, batch in enumerate(SCREAMING_SNAKE_CASE     ):
						# We could avoid this line since we set the accelerator with `device_placement=True`.
						batch.to(accelerator.device     )
						lowercase__            =      model(**SCREAMING_SNAKE_CASE     )
						lowercase__            =      outputs.loss
						lowercase__            =      loss / gradient_accumulation_steps
						accelerator.backward(SCREAMING_SNAKE_CASE     )
						if step % gradient_accumulation_steps == 0:
								optimizer.step()
								lr_scheduler.step()
								optimizer.zero_grad()

				model.eval()
				for step, batch in enumerate(SCREAMING_SNAKE_CASE     ):
						# We could avoid this line since we set the accelerator with `device_placement=True`.
						batch.to(accelerator.device     )
						with torch.no_grad():
								lowercase__            =      model(**SCREAMING_SNAKE_CASE     )
						lowercase__            =      outputs.logits.argmax(dim=-1     )
						lowercase__     ,		lowercase__            =      accelerator.gather_for_metrics((predictions, batch['''labels'''])     )
						metric.add_batch(
						    predictions=SCREAMING_SNAKE_CASE						,				references=SCREAMING_SNAKE_CASE						,				)

				lowercase__            =      metric.compute()
				# Use accelerator.print to print only on the main process.
				accelerator.print(f'epoch {epoch}:'						,				SCREAMING_SNAKE_CASE     )
def 			_a (      ):


		"""simple docstring"""




		lowercase__            =      argparse.ArgumentParser(description='''Simple example of training script.'''     )
		parser.add_argument(
		    '''--mixed_precision'''						,				type=SCREAMING_SNAKE_CASE						,				default=SCREAMING_SNAKE_CASE						,				choices=['''no''', '''fp16''', '''bf16''', '''fp8''']						,				help='''Whether to use mixed precision. Choose'''
		    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
		    '''and an Nvidia Ampere GPU.'''						,				)
		parser.add_argument('''--cpu'''						,				action='''store_true'''						,				help='''If passed, will train on the CPU.'''     )
		lowercase__            =      parser.parse_args()
		lowercase__            =      {'''lr''': 2E-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
		training_function(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     )


if __name__ == "__main__":
			main()






import argparse
import glob
import logging
import os
from argparse import Namespace
from importlib import import_module

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from seqeval.metrics import accuracy_score, fa_score, precision_score, recall_score
from torch.nn import CrossEntropyLoss
from torch.utils.data import DataLoader, TensorDataset
from utils_ner import TokenClassificationTask


lowerCAmelCase       =							logging.getLogger(__name__)


class        _a							(			UpperCamelCase__			):
							_lowercase	:			Union[str, Any]        = '''token-classification'''

							def __init__(							self:	int    ,						UpperCamelCase_:	Optional[Any]							)			->   Dict:





									"""simple docstring"""







									if type(UpperCamelCase_							) == dict:
											lowercase__            =      Namespace(**UpperCamelCase_							)
									lowercase__            =      import_module('''tasks'''							)
									try:
											lowercase__            =      getattr(UpperCamelCase_    ,						hparams.task_type							)
											lowercase__            =      token_classification_task_clazz()
									except AttributeError:
											raise ValueError(
											    f'Task {hparams.task_type} needs to be defined as a TokenClassificationTask subclass in {module}. '
											    f'Available tasks classes are: {TokenClassificationTask.__subclasses__()}'							)
									lowercase__            =      self.token_classification_task.get_labels(hparams.labels							)
									lowercase__            =      CrossEntropyLoss().ignore_index
									super().__init__(UpperCamelCase_    ,						len(self.labels							)    ,						self.mode							)

							def       lowerCamelCase_ (							self:	Tuple    ,						**UpperCamelCase_:	Optional[int]							)			->   str:





									"""simple docstring"""







									return self.model(**UpperCamelCase_							)

							def       lowerCamelCase_ (							self:	List[str]    ,						UpperCamelCase_:	List[Any]    ,						UpperCamelCase_:	int							)			->   int:





									"""simple docstring"""







									lowercase__            =      {'''input_ids''': batch[0], '''attention_mask''': batch[1], '''labels''': batch[3]}
									if self.config.model_type != "distilbert":
											lowercase__            =      (
											    batch[2] if self.config.model_type in ['''bert''', '''xlnet'''] else None
											)  # XLM and RoBERTa don"t use token_type_ids

									lowercase__            =      self(**UpperCamelCase_							)
									lowercase__            =      outputs[0]
									# tensorboard_logs = {"loss": loss, "rate": self.lr_scheduler.get_last_lr()[-1]}
									return {"loss": loss}

							def       lowerCamelCase_ (							self:	Optional[Any]							)			->   Optional[int]:





									"""simple docstring"""







									lowercase__            =      self.hparams
									for mode in ["train", "dev", "test"]:
											lowercase__            =      self._feature_file(UpperCamelCase_							)
											if os.path.exists(UpperCamelCase_							) and not args.overwrite_cache:
													logger.info('''Loading features from cached file %s'''    ,						UpperCamelCase_							)
													lowercase__            =      torch.load(UpperCamelCase_							)
											else:
													logger.info('''Creating features from dataset file at %s'''    ,						args.data_dir							)
													lowercase__            =      self.token_classification_task.read_examples_from_file(args.data_dir    ,						UpperCamelCase_							)
													lowercase__            =      self.token_classification_task.convert_examples_to_features(
													    UpperCamelCase_    ,						self.labels    ,						args.max_seq_length    ,						self.tokenizer    ,						cls_token_at_end=bool(self.config.model_type in ['''xlnet''']							)    ,						cls_token=self.tokenizer.cls_token    ,						cls_token_segment_id=2 if self.config.model_type in ['''xlnet'''] else 0    ,						sep_token=self.tokenizer.sep_token    ,						sep_token_extra=UpperCamelCase_    ,						pad_on_left=bool(self.config.model_type in ['''xlnet''']							)    ,						pad_token=self.tokenizer.pad_token_id    ,						pad_token_segment_id=self.tokenizer.pad_token_type_id    ,						pad_token_label_id=self.pad_token_label_id    ,						)
													logger.info('''Saving features into cached file %s'''    ,						UpperCamelCase_							)
													torch.save(UpperCamelCase_    ,						UpperCamelCase_							)

							def       lowerCamelCase_ (							self:	Union[str, Any]    ,						UpperCamelCase_:	int    ,						UpperCamelCase_:	int    ,						UpperCamelCase_:	bool = False							)			->   DataLoader:





									"""simple docstring"""







									lowercase__            =      self._feature_file(UpperCamelCase_							)
									logger.info('''Loading features from cached file %s'''    ,						UpperCamelCase_							)
									lowercase__            =      torch.load(UpperCamelCase_							)
									lowercase__            =      torch.tensor([f.input_ids for f in features]    ,						dtype=torch.long							)
									lowercase__            =      torch.tensor([f.attention_mask for f in features]    ,						dtype=torch.long							)
									if features[0].token_type_ids is not None:
											lowercase__            =      torch.tensor([f.token_type_ids for f in features]    ,						dtype=torch.long							)
									else:
											lowercase__            =      torch.tensor([0 for f in features]    ,						dtype=torch.long							)
											# HACK(we will not use this anymore soon)
									lowercase__            =      torch.tensor([f.label_ids for f in features]    ,						dtype=torch.long							)
									return DataLoader(
									    TensorDataset(UpperCamelCase_    ,						UpperCamelCase_    ,						UpperCamelCase_    ,						UpperCamelCase_							)    ,						batch_size=UpperCamelCase_							)

							def       lowerCamelCase_ (							self:	Any    ,						UpperCamelCase_:	int    ,						UpperCamelCase_:	List[Any]							)			->   Union[str, Any]:





									"""simple docstring"""







									"""Compute validation""" ""
									lowercase__            =      {'''input_ids''': batch[0], '''attention_mask''': batch[1], '''labels''': batch[3]}
									if self.config.model_type != "distilbert":
											lowercase__            =      (
											    batch[2] if self.config.model_type in ['''bert''', '''xlnet'''] else None
											)  # XLM and RoBERTa don"t use token_type_ids
									lowercase__            =      self(**UpperCamelCase_							)
									lowercase__     ,		lowercase__            =      outputs[:2]
									lowercase__            =      logits.detach().cpu().numpy()
									lowercase__            =      inputs['''labels'''].detach().cpu().numpy()
									return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}

							def       lowerCamelCase_ (							self:	Dict    ,						UpperCamelCase_:	List[str]							)			->   int:





									"""simple docstring"""







									lowercase__            =      torch.stack([x['''val_loss'''] for x in outputs]							).mean()
									lowercase__            =      np.concatenate([x['''pred'''] for x in outputs]    ,						axis=0							)
									lowercase__            =      np.argmax(UpperCamelCase_    ,						axis=2							)
									lowercase__            =      np.concatenate([x['''target'''] for x in outputs]    ,						axis=0							)

									lowercase__            =      dict(enumerate(self.labels							)							)
									lowercase__            =      [[] for _ in range(out_label_ids.shape[0]							)]
									lowercase__            =      [[] for _ in range(out_label_ids.shape[0]							)]

									for i in range(out_label_ids.shape[0]							):
											for j in range(out_label_ids.shape[1]							):
													if out_label_ids[i, j] != self.pad_token_label_id:
															out_label_list[i].append(label_map[out_label_ids[i][j]]							)
															preds_list[i].append(label_map[preds[i][j]]							)

									lowercase__            =      {
									    '''val_loss''': val_loss_mean,
									    '''accuracy_score''': accuracy_score(UpperCamelCase_    ,						UpperCamelCase_							),
									    '''precision''': precision_score(UpperCamelCase_    ,						UpperCamelCase_							),
									    '''recall''': recall_score(UpperCamelCase_    ,						UpperCamelCase_							),
									    '''f1''': fa_score(UpperCamelCase_    ,						UpperCamelCase_							),
									}

									lowercase__            =      dict(results.items()							)
									lowercase__            =      results
									return ret, preds_list, out_label_list

							def       lowerCamelCase_ (							self:	List[Any]    ,						UpperCamelCase_:	List[Any]							)			->   Dict:





									"""simple docstring"""







									lowercase__     ,		lowercase__     ,		lowercase__            =      self._eval_end(UpperCamelCase_							)
									lowercase__            =      ret['''log''']
									return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}

							def       lowerCamelCase_ (							self:	List[str]    ,						UpperCamelCase_:	Dict							)			->   Dict:





									"""simple docstring"""







									lowercase__     ,		lowercase__     ,		lowercase__            =      self._eval_end(UpperCamelCase_							)

									# Converting to the dict required by pl
									# https://github.com/PyTorchLightning/pytorch-lightning/blob/master/\
									# pytorch_lightning/trainer/logging.py#L139
									lowercase__            =      ret['''log''']
									# `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
									return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}


							@staticmethod
							def       lowerCamelCase_ (							UpperCamelCase_:	Optional[Any]    ,						UpperCamelCase_:	str							)			->   Optional[Any]:





									"""simple docstring"""







									BaseTransformer.add_model_specific_args(UpperCamelCase_    ,						UpperCamelCase_							)
									parser.add_argument(
									    '''--task_type'''    ,						default='''NER'''    ,						type=UpperCamelCase_    ,						help='''Task type to fine tune in training (e.g. NER, POS, etc)'''							)
									parser.add_argument(
									    '''--max_seq_length'''    ,						default=128    ,						type=UpperCamelCase_    ,						help=(
									        '''The maximum total input sequence length after tokenization. Sequences longer '''
									        '''than this will be truncated, sequences shorter will be padded.'''
									    )    ,						)

									parser.add_argument(
									    '''--labels'''    ,						default=''''''    ,						type=UpperCamelCase_    ,						help='''Path to a file containing all labels. If not specified, CoNLL-2003 labels are used.'''    ,						)
									parser.add_argument(
									    '''--gpus'''    ,						default=0    ,						type=UpperCamelCase_    ,						help='''The number of GPUs allocated for this, it is by default 0 meaning none'''    ,						)

									parser.add_argument(
									    '''--overwrite_cache'''    ,						action='''store_true'''    ,						help='''Overwrite the cached training and evaluation sets'''							)

									return parser


if __name__ == "__main__":
			lowerCAmelCase       =							argparse.ArgumentParser()
			add_generic_args(parser, os.getcwd())
			lowerCAmelCase       =							NERTransformer.add_model_specific_args(parser, os.getcwd())
			lowerCAmelCase       =							parser.parse_args()
			lowerCAmelCase       =							NERTransformer(args)
			lowerCAmelCase       =							generic_train(model, args)

			if args.do_predict:
						# See https://github.com/huggingface/transformers/issues/3159
						# pl use this default format to create a checkpoint:
						# https://github.com/PyTorchLightning/pytorch-lightning/blob/master\
						# /pytorch_lightning/callbacks/model_checkpoint.py#L322
						lowerCAmelCase       =							sorted(glob.glob(os.path.join(args.output_dir, 'checkpoint-epoch=*.ckpt'), recursive=True))
						lowerCAmelCase       =							model.load_from_checkpoint(checkpoints[-1])
						trainer.test(model)


