            total_loss += last_loss
            acc_loss += last_loss
            accelerator.backward(loss)
            del batch, loss, outputs
            torch.cuda.empty_cache()

            accelerator.clip_grad_norm_(model.parameters(), max_grad_norm)
            optimizer.step()
            lr_scheduler.step()
            optimizer.zero_grad()
            progress_bar.update(1)
