def train(self, train_dataset, output_dir, show_running_loss=True, eval_data=None, verbose=True, **kwargs):
    '\n        Trains the model on train_dataset.\n\n        Utility function to be used by the train_model() method. Not intended to be used directly.\n        '
    model = self.model
    args = self.args
    tb_writer = SummaryWriter(logdir=args.tensorboard_dir)
    train_sampler = RandomSampler(train_dataset)
    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size, num_workers=self.args.dataloader_num_workers)
    if (args.max_steps > 0):
        t_total = args.max_steps
        args.num_train_epochs = ((args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps)) + 1)
    else:
        t_total = ((len(train_dataloader) // args.gradient_accumulation_steps) * args.num_train_epochs)
    no_decay = ['bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = []
    custom_parameter_names = set()
    for group in self.args.custom_parameter_groups:
        params = group.pop('params')
        custom_parameter_names.update(params)
        param_group = {**group}
        param_group['params'] = [p for (n, p) in model.named_parameters() if (n in params)]
        optimizer_grouped_parameters.append(param_group)
    for group in self.args.custom_layer_parameters:
        layer_number = group.pop('layer')
        layer = f'layer.{layer_number}.'
        group_d = {**group}
        group_nd = {**group}
        group_nd['weight_decay'] = 0.0
        params_d = []
        params_nd = []
        for (n, p) in model.named_parameters():
            if ((n not in custom_parameter_names) and (layer in n)):
                if any(((nd in n) for nd in no_decay)):
                    params_nd.append(p)
                else:
                    params_d.append(p)
                custom_parameter_names.add(n)
        group_d['params'] = params_d
        group_nd['params'] = params_nd
        optimizer_grouped_parameters.append(group_d)
        optimizer_grouped_parameters.append(group_nd)
    if (not self.args.train_custom_parameters_only):
        optimizer_grouped_parameters.extend([{'params': [p for (n, p) in model.named_parameters() if ((n not in custom_parameter_names) and (not any(((nd in n) for nd in no_decay))))], 'weight_decay': args.weight_decay}, {'params': [p for (n, p) in model.named_parameters() if ((n not in custom_parameter_names) and any(((nd in n) for nd in no_decay)))], 'weight_decay': 0.0}])
    warmup_steps = math.ceil((t_total * args.warmup_ratio))
    args.warmup_steps = (warmup_steps if (args.warmup_steps == 0) else args.warmup_steps)
    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
    if (args.model_name and os.path.isfile(os.path.join(args.model_name, 'optimizer.pt')) and os.path.isfile(os.path.join(args.model_name, 'scheduler.pt'))):
        optimizer.load_state_dict(torch.load(os.path.join(args.model_name, 'optimizer.pt')))
        scheduler.load_state_dict(torch.load(os.path.join(args.model_name, 'scheduler.pt')))
    if (args.n_gpu > 1):
        model = torch.nn.DataParallel(model)
    logger.info(' Training started')
    global_step = 0
    (tr_loss, logging_loss) = (0.0, 0.0)
    model.zero_grad()
    train_iterator = trange(int(args.num_train_epochs), desc='Epoch', disable=args.silent, mininterval=0)
    epoch_number = 0
    best_eval_metric = None
    early_stopping_counter = 0
    steps_trained_in_current_epoch = 0
    epochs_trained = 0
    if (args.model_name and os.path.exists(args.model_name)):
        try:
            checkpoint_suffix = args.model_name.split('/')[(- 1)].split('-')
            if (len(checkpoint_suffix) > 2):
                checkpoint_suffix = checkpoint_suffix[1]
            else:
                checkpoint_suffix = checkpoint_suffix[(- 1)]
            global_step = int(checkpoint_suffix)
            epochs_trained = (global_step // (len(train_dataloader) // args.gradient_accumulation_steps))
            steps_trained_in_current_epoch = (global_step % (len(train_dataloader) // args.gradient_accumulation_steps))
            logger.info('   Continuing training from checkpoint, will skip to saved global_step')
            logger.info('   Continuing training from epoch %d', epochs_trained)
            logger.info('   Continuing training from global step %d', global_step)
            logger.info('   Will skip the first %d steps in the current epoch', steps_trained_in_current_epoch)
        except ValueError:
            logger.info('   Starting fine-tuning.')
    if args.evaluate_during_training:
        training_progress_scores = self._create_training_progress_scores(**kwargs)
    if args.wandb_project:
        wandb.init(project=args.wandb_project, config={**asdict(args)}, **args.wandb_kwargs)
        wandb.watch(self.model)
    if args.fp16:
        from torch.cuda import amp
        scaler = amp.GradScaler()
    model.train()
    for current_epoch in train_iterator:
        if (epochs_trained > 0):
            epochs_trained -= 1
            continue
        train_iterator.set_description(f'Epoch {(epoch_number + 1)} of {args.num_train_epochs}')
        batch_iterator = tqdm(train_dataloader, desc=f'Running Epoch {epoch_number} of {args.num_train_epochs}', disable=args.silent, mininterval=0)
        for (step, batch) in enumerate(batch_iterator):
            if (steps_trained_in_current_epoch > 0):
                steps_trained_in_current_epoch -= 1
                continue
            inputs = self._get_inputs_dict(batch)
            if args.fp16:
                with amp.autocast():
                    outputs = model(**inputs)
                    loss = outputs[0]
            else:
                outputs = model(**inputs)
                loss = outputs[0]
            if (args.n_gpu > 1):
                loss = loss.mean()
            current_loss = loss.item()
            if show_running_loss:
                batch_iterator.set_description(f'Epochs {epoch_number}/{args.num_train_epochs}. Running Loss: {current_loss:9.4f}')
            if (args.gradient_accumulation_steps > 1):
                loss = (loss / args.gradient_accumulation_steps)
            if args.fp16:
                scaler.scale(loss).backward()
            else:
                loss.backward()
            tr_loss += loss.item()
            if (((step + 1) % args.gradient_accumulation_steps) == 0):
                if args.fp16:
                    scaler.unscale_(optimizer)
                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
                if args.fp16:
                    scaler.step(optimizer)
                    scaler.update()
                else:
                    optimizer.step()
                scheduler.step()
                model.zero_grad()
                global_step += 1
                if ((args.logging_steps > 0) and ((global_step % args.logging_steps) == 0)):
                    tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step)
                    tb_writer.add_scalar('loss', ((tr_loss - logging_loss) / args.logging_steps), global_step)
                    logging_loss = tr_loss
                    if args.wandb_project:
                        wandb.log({'Training loss': current_loss, 'lr': scheduler.get_lr()[0], 'global_step': global_step})
                if ((args.save_steps > 0) and ((global_step % args.save_steps) == 0)):
                    output_dir_current = os.path.join(output_dir, 'checkpoint-{}'.format(global_step))
                    self._save_model(output_dir_current, optimizer, scheduler, model=model)
                if (args.evaluate_during_training and ((args.evaluate_during_training_steps > 0) and ((global_step % args.evaluate_during_training_steps) == 0))):
                    results = self.eval_model(eval_data, verbose=(verbose and args.evaluate_during_training_verbose), silent=args.evaluate_during_training_silent, **kwargs)
                    for (key, value) in results.items():
                        tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
                    output_dir_current = os.path.join(output_dir, 'checkpoint-{}'.format(global_step))
                    if args.save_eval_checkpoints:
                        self._save_model(output_dir_current, optimizer, scheduler, model=model, results=results)
                    training_progress_scores['global_step'].append(global_step)
                    training_progress_scores['train_loss'].append(current_loss)
                    for key in results:
                        training_progress_scores[key].append(results[key])
                    report = pd.DataFrame(training_progress_scores)
                    report.to_csv(os.path.join(args.output_dir, 'training_progress_scores.csv'), index=False)
                    if args.wandb_project:
                        wandb.log(self._get_last_metrics(training_progress_scores))
                    if (not best_eval_metric):
                        best_eval_metric = results[args.early_stopping_metric]
                        if args.save_best_model:
                            self._save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results)
                    if (best_eval_metric and args.early_stopping_metric_minimize):
                        if ((results[args.early_stopping_metric] - best_eval_metric) < args.early_stopping_delta):
                            best_eval_metric = results[args.early_stopping_metric]
                            if args.save_best_model:
                                self._save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results)
                            early_stopping_counter = 0
                        elif args.use_early_stopping:
                            if (early_stopping_counter < args.early_stopping_patience):
                                early_stopping_counter += 1
                                if verbose:
                                    logger.info(f' No improvement in {args.early_stopping_metric}')
                                    logger.info(f' Current step: {early_stopping_counter}')
                                    logger.info(f' Early stopping patience: {args.early_stopping_patience}')
                            else:
                                if verbose:
                                    logger.info(f' Patience of {args.early_stopping_patience} steps reached')
                                    logger.info(' Training terminated.')
                                    train_iterator.close()
                                return (global_step, (tr_loss / global_step))
                    elif ((results[args.early_stopping_metric] - best_eval_metric) > args.early_stopping_delta):
                        best_eval_metric = results[args.early_stopping_metric]
                        if args.save_best_model:
                            self._save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results)
                        early_stopping_counter = 0
                    elif args.use_early_stopping:
                        if (early_stopping_counter < args.early_stopping_patience):
                            early_stopping_counter += 1
                            if verbose:
                                logger.info(f' No improvement in {args.early_stopping_metric}')
                                logger.info(f' Current step: {early_stopping_counter}')
                                logger.info(f' Early stopping patience: {args.early_stopping_patience}')
                        else:
                            if verbose:
                                logger.info(f' Patience of {args.early_stopping_patience} steps reached')
                                logger.info(' Training terminated.')
                                train_iterator.close()
                            return (global_step, (tr_loss / global_step))
        epoch_number += 1
        output_dir_current = os.path.join(output_dir, 'checkpoint-{}-epoch-{}'.format(global_step, epoch_number))
        if (args.save_model_every_epoch or args.evaluate_during_training):
            os.makedirs(output_dir_current, exist_ok=True)
        if args.save_model_every_epoch:
            self._save_model(output_dir_current, optimizer, scheduler, model=model)
        if args.evaluate_during_training:
            results = self.eval_model(eval_data, verbose=(verbose and args.evaluate_during_training_verbose), silent=args.evaluate_during_training_silent, **kwargs)
            if args.save_eval_checkpoints:
                self._save_model(output_dir_current, optimizer, scheduler, results=results)
            training_progress_scores['global_step'].append(global_step)
            training_progress_scores['train_loss'].append(current_loss)
            for key in results:
                training_progress_scores[key].append(results[key])
            report = pd.DataFrame(training_progress_scores)
            report.to_csv(os.path.join(args.output_dir, 'training_progress_scores.csv'), index=False)
            if args.wandb_project:
                wandb.log(self._get_last_metrics(training_progress_scores))
            if (not best_eval_metric):
                best_eval_metric = results[args.early_stopping_metric]
                if args.save_best_model:
                    self._save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results)
            if (best_eval_metric and args.early_stopping_metric_minimize):
                if ((results[args.early_stopping_metric] - best_eval_metric) < args.early_stopping_delta):
                    best_eval_metric = results[args.early_stopping_metric]
                    if args.save_best_model:
                        self._save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results)
                    early_stopping_counter = 0
                elif (args.use_early_stopping and args.early_stopping_consider_epochs):
                    if (early_stopping_counter < args.early_stopping_patience):
                        early_stopping_counter += 1
                        if verbose:
                            logger.info(f' No improvement in {args.early_stopping_metric}')
                            logger.info(f' Current step: {early_stopping_counter}')
                            logger.info(f' Early stopping patience: {args.early_stopping_patience}')
                    else:
                        if verbose:
                            logger.info(f' Patience of {args.early_stopping_patience} steps reached')
                            logger.info(' Training terminated.')
                            train_iterator.close()
                        return (global_step, (tr_loss / global_step))
            elif ((results[args.early_stopping_metric] - best_eval_metric) > args.early_stopping_delta):
                best_eval_metric = results[args.early_stopping_metric]
                if args.save_best_model:
                    self._save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results)
                early_stopping_counter = 0
            elif (args.use_early_stopping and args.early_stopping_consider_epochs):
                if (early_stopping_counter < args.early_stopping_patience):
                    early_stopping_counter += 1
                    if verbose:
                        logger.info(f' No improvement in {args.early_stopping_metric}')
                        logger.info(f' Current step: {early_stopping_counter}')
                        logger.info(f' Early stopping patience: {args.early_stopping_patience}')
                else:
                    if verbose:
                        logger.info(f' Patience of {args.early_stopping_patience} steps reached')
                        logger.info(' Training terminated.')
                        train_iterator.close()
                    return (global_step, (tr_loss / global_step))
    return (global_step, (tr_loss / global_step))
Trains the model on train_dataset.

Utility function to be used by the train_model() method. Not intended to be used directly.
def train(self, train_dataset, output_dir, show_running_loss=True, eval_data=None, verbose=True, **kwargs):
    '\n        Trains the model on train_dataset.\n\n        Utility function to be used by the train_model() method. Not intended to be used directly.\n        '
    model = self.model
    args = self.args
    tb_writer = SummaryWriter(logdir=args.tensorboard_dir)
    train_sampler = RandomSampler(train_dataset)
    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size, num_workers=self.args.dataloader_num_workers)
    if (args.max_steps > 0):
        t_total = args.max_steps
        args.num_train_epochs = ((args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps)) + 1)
    else:
        t_total = ((len(train_dataloader) // args.gradient_accumulation_steps) * args.num_train_epochs)
    no_decay = ['bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = []
    custom_parameter_names = set()
    for group in self.args.custom_parameter_groups:
        params = group.pop('params')
        custom_parameter_names.update(params)
        param_group = {**group}
        param_group['params'] = [p for (n, p) in model.named_parameters() if (n in params)]
        optimizer_grouped_parameters.append(param_group)
    for group in self.args.custom_layer_parameters:
        layer_number = group.pop('layer')
        layer = f'layer.{layer_number}.'
        group_d = {**group}
        group_nd = {**group}
        group_nd['weight_decay'] = 0.0
        params_d = []
        params_nd = []
        for (n, p) in model.named_parameters():
            if ((n not in custom_parameter_names) and (layer in n)):
                if any(((nd in n) for nd in no_decay)):
                    params_nd.append(p)
                else:
                    params_d.append(p)
                custom_parameter_names.add(n)
        group_d['params'] = params_d
        group_nd['params'] = params_nd
        optimizer_grouped_parameters.append(group_d)
        optimizer_grouped_parameters.append(group_nd)
    if (not self.args.train_custom_parameters_only):
        optimizer_grouped_parameters.extend([{'params': [p for (n, p) in model.named_parameters() if ((n not in custom_parameter_names) and (not any(((nd in n) for nd in no_decay))))], 'weight_decay': args.weight_decay}, {'params': [p for (n, p) in model.named_parameters() if ((n not in custom_parameter_names) and any(((nd in n) for nd in no_decay)))], 'weight_decay': 0.0}])
    warmup_steps = math.ceil((t_total * args.warmup_ratio))
    args.warmup_steps = (warmup_steps if (args.warmup_steps == 0) else args.warmup_steps)
    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
    if (args.model_name and os.path.isfile(os.path.join(args.model_name, 'optimizer.pt')) and os.path.isfile(os.path.join(args.model_name, 'scheduler.pt'))):
        optimizer.load_state_dict(torch.load(os.path.join(args.model_name, 'optimizer.pt')))
        scheduler.load_state_dict(torch.load(os.path.join(args.model_name, 'scheduler.pt')))
    if (args.n_gpu > 1):
        model = torch.nn.DataParallel(model)
    logger.info(' Training started')
    global_step = 0
    (tr_loss, logging_loss) = (0.0, 0.0)
    model.zero_grad()
    train_iterator = trange(int(args.num_train_epochs), desc='Epoch', disable=args.silent, mininterval=0)
    epoch_number = 0
    best_eval_metric = None
    early_stopping_counter = 0
    steps_trained_in_current_epoch = 0
    epochs_trained = 0
    if (args.model_name and os.path.exists(args.model_name)):
        try:
            checkpoint_suffix = args.model_name.split('/')[(- 1)].split('-')
            if (len(checkpoint_suffix) > 2):
                checkpoint_suffix = checkpoint_suffix[1]
            else:
                checkpoint_suffix = checkpoint_suffix[(- 1)]
            global_step = int(checkpoint_suffix)
            epochs_trained = (global_step // (len(train_dataloader) // args.gradient_accumulation_steps))
            steps_trained_in_current_epoch = (global_step % (len(train_dataloader) // args.gradient_accumulation_steps))
            logger.info('   Continuing training from checkpoint, will skip to saved global_step')
            logger.info('   Continuing training from epoch %d', epochs_trained)
            logger.info('   Continuing training from global step %d', global_step)
            logger.info('   Will skip the first %d steps in the current epoch', steps_trained_in_current_epoch)
        except ValueError:
            logger.info('   Starting fine-tuning.')
    if args.evaluate_during_training:
        training_progress_scores = self._create_training_progress_scores(**kwargs)
    if args.wandb_project:
        wandb.init(project=args.wandb_project, config={**asdict(args)}, **args.wandb_kwargs)
        wandb.watch(self.model)
    if args.fp16:
        from torch.cuda import amp
        scaler = amp.GradScaler()
    model.train()
    for current_epoch in train_iterator:
        if (epochs_trained > 0):
            epochs_trained -= 1
            continue
        train_iterator.set_description(f'Epoch {(epoch_number + 1)} of {args.num_train_epochs}')
        batch_iterator = tqdm(train_dataloader, desc=f'Running Epoch {epoch_number} of {args.num_train_epochs}', disable=args.silent, mininterval=0)
        for (step, batch) in enumerate(batch_iterator):
            if (steps_trained_in_current_epoch > 0):
                steps_trained_in_current_epoch -= 1
                continue
            inputs = self._get_inputs_dict(batch)
            if args.fp16:
                with amp.autocast():
                    outputs = model(**inputs)
                    loss = outputs[0]
            else:
                outputs = model(**inputs)
                loss = outputs[0]
            if (args.n_gpu > 1):
                loss = loss.mean()
            current_loss = loss.item()
            if show_running_loss:
                batch_iterator.set_description(f'Epochs {epoch_number}/{args.num_train_epochs}. Running Loss: {current_loss:9.4f}')
            if (args.gradient_accumulation_steps > 1):
                loss = (loss / args.gradient_accumulation_steps)
            if args.fp16:
                scaler.scale(loss).backward()
            else:
                loss.backward()
            tr_loss += loss.item()
            if (((step + 1) % args.gradient_accumulation_steps) == 0):
                if args.fp16:
                    scaler.unscale_(optimizer)
                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
                if args.fp16:
                    scaler.step(optimizer)
                    scaler.update()
                else:
                    optimizer.step()
                scheduler.step()
                model.zero_grad()
                global_step += 1
                if ((args.logging_steps > 0) and ((global_step % args.logging_steps) == 0)):
                    tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step)
                    tb_writer.add_scalar('loss', ((tr_loss - logging_loss) / args.logging_steps), global_step)
                    logging_loss = tr_loss
                    if args.wandb_project:
                        wandb.log({'Training loss': current_loss, 'lr': scheduler.get_lr()[0], 'global_step': global_step})
                if ((args.save_steps > 0) and ((global_step % args.save_steps) == 0)):
                    output_dir_current = os.path.join(output_dir, 'checkpoint-{}'.format(global_step))
                    self._save_model(output_dir_current, optimizer, scheduler, model=model)
                if (args.evaluate_during_training and ((args.evaluate_during_training_steps > 0) and ((global_step % args.evaluate_during_training_steps) == 0))):
                    results = self.eval_model(eval_data, verbose=(verbose and args.evaluate_during_training_verbose), silent=args.evaluate_during_training_silent, **kwargs)
                    for (key, value) in results.items():
                        tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
                    output_dir_current = os.path.join(output_dir, 'checkpoint-{}'.format(global_step))
                    if args.save_eval_checkpoints:
                        self._save_model(output_dir_current, optimizer, scheduler, model=model, results=results)
                    training_progress_scores['global_step'].append(global_step)
                    training_progress_scores['train_loss'].append(current_loss)
                    for key in results:
                        training_progress_scores[key].append(results[key])
                    report = pd.DataFrame(training_progress_scores)
                    report.to_csv(os.path.join(args.output_dir, 'training_progress_scores.csv'), index=False)
                    if args.wandb_project:
                        wandb.log(self._get_last_metrics(training_progress_scores))
                    if (not best_eval_metric):
                        best_eval_metric = results[args.early_stopping_metric]
                        if args.save_best_model:
                            self._save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results)
                    if (best_eval_metric and args.early_stopping_metric_minimize):
                        if ((results[args.early_stopping_metric] - best_eval_metric) < args.early_stopping_delta):
                            best_eval_metric = results[args.early_stopping_metric]
                            if args.save_best_model:
                                self._save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results)
                            early_stopping_counter = 0
                        elif args.use_early_stopping:
                            if (early_stopping_counter < args.early_stopping_patience):
                                early_stopping_counter += 1
                                if verbose:
                                    logger.info(f' No improvement in {args.early_stopping_metric}')
                                    logger.info(f' Current step: {early_stopping_counter}')
                                    logger.info(f' Early stopping patience: {args.early_stopping_patience}')
                            else:
                                if verbose:
                                    logger.info(f' Patience of {args.early_stopping_patience} steps reached')
                                    logger.info(' Training terminated.')
                                    train_iterator.close()
                                return (global_step, (tr_loss / global_step))
                    elif ((results[args.early_stopping_metric] - best_eval_metric) > args.early_stopping_delta):
                        best_eval_metric = results[args.early_stopping_metric]
                        if args.save_best_model:
                            self._save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results)
                        early_stopping_counter = 0
                    elif args.use_early_stopping:
                        if (early_stopping_counter < args.early_stopping_patience):
                            early_stopping_counter += 1
                            if verbose:
                                logger.info(f' No improvement in {args.early_stopping_metric}')
                                logger.info(f' Current step: {early_stopping_counter}')
                                logger.info(f' Early stopping patience: {args.early_stopping_patience}')
                        else:
                            if verbose:
                                logger.info(f' Patience of {args.early_stopping_patience} steps reached')
                                logger.info(' Training terminated.')
                                train_iterator.close()
                            return (global_step, (tr_loss / global_step))
        epoch_number += 1
        output_dir_current = os.path.join(output_dir, 'checkpoint-{}-epoch-{}'.format(global_step, epoch_number))
        if (args.save_model_every_epoch or args.evaluate_during_training):
            os.makedirs(output_dir_current, exist_ok=True)
        if args.save_model_every_epoch:
            self._save_model(output_dir_current, optimizer, scheduler, model=model)
        if args.evaluate_during_training:
            results = self.eval_model(eval_data, verbose=(verbose and args.evaluate_during_training_verbose), silent=args.evaluate_during_training_silent, **kwargs)
            if args.save_eval_checkpoints:
                self._save_model(output_dir_current, optimizer, scheduler, results=results)
            training_progress_scores['global_step'].append(global_step)
            training_progress_scores['train_loss'].append(current_loss)
            for key in results:
                training_progress_scores[key].append(results[key])
            report = pd.DataFrame(training_progress_scores)
            report.to_csv(os.path.join(args.output_dir, 'training_progress_scores.csv'), index=False)
            if args.wandb_project:
                wandb.log(self._get_last_metrics(training_progress_scores))
            if (not best_eval_metric):
                best_eval_metric = results[args.early_stopping_metric]
                if args.save_best_model:
                    self._save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results)
            if (best_eval_metric and args.early_stopping_metric_minimize):
                if ((results[args.early_stopping_metric] - best_eval_metric) < args.early_stopping_delta):
                    best_eval_metric = results[args.early_stopping_metric]
                    if args.save_best_model:
                        self._save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results)
                    early_stopping_counter = 0
                elif (args.use_early_stopping and args.early_stopping_consider_epochs):
                    if (early_stopping_counter < args.early_stopping_patience):
                        early_stopping_counter += 1
                        if verbose:
                            logger.info(f' No improvement in {args.early_stopping_metric}')
                            logger.info(f' Current step: {early_stopping_counter}')
                            logger.info(f' Early stopping patience: {args.early_stopping_patience}')
                    else:
                        if verbose:
                            logger.info(f' Patience of {args.early_stopping_patience} steps reached')
                            logger.info(' Training terminated.')
                            train_iterator.close()
                        return (global_step, (tr_loss / global_step))
            elif ((results[args.early_stopping_metric] - best_eval_metric) > args.early_stopping_delta):
                best_eval_metric = results[args.early_stopping_metric]
                if args.save_best_model:
                    self._save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results)
                early_stopping_counter = 0
            elif (args.use_early_stopping and args.early_stopping_consider_epochs):
                if (early_stopping_counter < args.early_stopping_patience):
                    early_stopping_counter += 1
                    if verbose:
                        logger.info(f' No improvement in {args.early_stopping_metric}')
                        logger.info(f' Current step: {early_stopping_counter}')
                        logger.info(f' Early stopping patience: {args.early_stopping_patience}')
                else:
                    if verbose:
                        logger.info(f' Patience of {args.early_stopping_patience} steps reached')
                        logger.info(' Training terminated.')
                        train_iterator.close()
                    return (global_step, (tr_loss / global_step))
    return (global_step, (tr_loss / global_step))