usrbinenv python codingutf8 2021 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license training a clip like dual encoder models using text and vision encoders in the library the script can be used to train clip like models for languages other than english by using a text encoder pretrained in the desired language currently this script support the following vision and text models vision models vithttps huggingface comodels filtervit clip https huggingface comodels filterclip text models bert roberta https huggingface comodels filterfillmask cache the result arguments pertaining to which modelconfigtokenizer we are going to finetune or train from scratch arguments pertaining to what data we are going to input our model for training and eval we use torchvision for faster image preprocessing we need to ensure faster processing speed as it can become a bottleneck on tpu dtaset for loading imagetext data for tasks like clip training image captioning args root string the root path where the dataset is stored filepath string path to the file containing the imagepaths and associated captions the expected format is jsonlines where each line is a json object containing to keys imagepath the path to the image captions an array of captions transform callable optional a functiontransform that takes in an pil image and returns a transformed version e g transforms totensor targettransform callable optional a functiontransform that takes in the target and transforms it transforms callable optional a functiontransform that takes input sample and its target as entry and returns a transformed version returns a linear warmup lineardecay learning rate function stepsperepoch traindssize trainbatchsize numtrainsteps stepsperepoch numtrainepochs warmupfn optax linearscheduleinitvalue0 0 endvaluelearningrate transitionstepsnumwarmupsteps decayfn optax linearschedule initvaluelearningrate endvalue0 transitionstepsnumtrainsteps numwarmupsteps schedulefn optax joinschedulesscheduleswarmupfn decayfn boundariesnumwarmupsteps return schedulefn def main parser hfargumentparsermodelarguments datatrainingarguments trainingarguments if lensys argv 2 and sys argv1 endswith json if we pass only one argument to the script and it s the path to a json file let s parse it to get our arguments modelargs dataargs trainingargs parser parsejsonfilejsonfileos path abspathsys argv1 else modelargs dataargs trainingargs parser parseargsintodataclasses if os path existstrainingargs outputdir and os listdirtrainingargs outputdir and trainingargs dotrain and not trainingargs overwriteoutputdir raise valueerror foutput directory trainingargs outputdir already exists and is not empty use overwriteoutputdir to overcome make one log on every process with the configuration for debugging logging basicconfig formatasctimes levelnames names messages datefmtmdy h m s levellogging info setup logging we only want one process per machine to log things on the screen logger setlevellogging info if jax processindex 0 else logging error if jax processindex 0 transformers utils logging setverbosityinfo else transformers utils logging setverbosityerror set the verbosity to info of the transformers logger on main process only logger infoftrainingevaluation parameters trainingargs if modelargs tokenizername tokenizer autotokenizer frompretrained modelargs tokenizername cachedirmodelargs cachedir usefastmodelargs usefasttokenizer elif modelargs textmodelnameorpath tokenizer autotokenizer frompretrained modelargs textmodelnameorpath cachedirmodelargs cachedir usefastmodelargs usefasttokenizer else raise valueerror you are instantiating a new tokenizer from scratch this is not supported by this script you can do it from another script save it and load it from here using tokenizername model flaxhybridclip fromtextvisionpretrained modelargs textmodelnameorpath modelargs visionmodelnameorpath seedtrainingargs seed dtypegetattrjnp modelargs dtype textfromptmodelargs frompt visionfromptmodelargs frompt config model config set seed for torch dataloaders setseedtrainingargs seed initialize torchvision transforms and jit them for faster processing preprocess transformconfig visionconfig imagesize preprocess torch jit scriptpreprocess initialize the imagetext dataset traindataset imagetextdataset dataargs datadir dataargs trainfile captionsperimage2 transformpreprocess evaldataset imagetextdataset dataargs datadir dataargs validationfile captionsperimage1 transformpreprocess store some constant numepochs inttrainingargs numtrainepochs trainbatchsize inttrainingargs perdevicetrainbatchsize jax devicecount evalbatchsize inttrainingargs perdeviceevalbatchsize jax devicecount stepsperepoch lentraindataset trainbatchsize totaltrainsteps stepsperepoch numepochs use collate function to tokenizer the text and convert the processed images to numpy def collatefnexamples pixelvalues torch stackexample0 for example in examples permute0 2 3 1 numpy captions example1 for example in examples inputs tokenizer captions maxlengthdataargs maxseqlength paddingmaxlength truncationtrue returntensorsnp batch pixelvalues pixelvalues inputids inputsinputids attentionmask inputsattentionmask return batch create data loaders trainloader torch utils data dataloader traindataset batchsizetrainbatchsize shuffletrue numworkersdataargs preprocessingnumworkers persistentworkerstrue droplasttrue collatefncollatefn evalloader torch utils data dataloader evaldataset batchsizeevalbatchsize shufflefalse numworkersdataargs preprocessingnumworkers persistentworkerstrue droplasttrue collatefncollatefn enable tensorboard only on the master node if hastensorboard and jax processindex 0 summarywriter summarywriterlogdirpathtrainingargs outputdir joinpathlogs asposix initialize our training rng jax random prngkeytrainingargs seed rng dropoutrng jax random splitrng create learning rate schedule lineardecaylrschedulefn createlearningratefn lentraindataset trainbatchsize trainingargs numtrainepochs trainingargs warmupsteps trainingargs learningrate create adam optimizer adamw optax adamw learningratelineardecaylrschedulefn b1trainingargs adambeta1 b2trainingargs adambeta2 epstrainingargs adamepsilon weightdecaytrainingargs weightdecay setup train state state trainstate createapplyfnmodel call paramsmodel params txadamw dropoutrngdropoutrng def crossentropylogits axis logprobs jax nn logsoftmaxlogits axisaxis nll jnp diaglogprobs ce jnp meannll return ce def cliplosssimilarity loss crossentropysimilarity axis0 crossentropysimilarity axis1 2 return loss define gradient update step fn def trainstepstate batch dropoutrng newdropoutrng jax random splitstate dropoutrng def computelossparams logits state applyfnbatch paramsparams dropoutrngdropoutrng traintrue0 loss cliplosslogits return loss gradfn jax valueandgradcomputeloss loss grad gradfnstate params grad jax lax pmeangrad batch newstate state applygradientsgradsgrad dropoutrngnewdropoutrng metrics loss loss learningrate lineardecaylrschedulefnstate step metrics jax lax pmeanmetrics axisnamebatch return newstate metrics define eval fn def evalstepparams batch logits modelbatch paramsparams trainfalse0 loss cliplosslogits summarize metrics metrics loss loss metrics jax lax pmeanmetrics axisnamebatch return metrics create parallel version of the train and eval step ptrainstep jax pmaptrainstep batch donateargnums0 pevalstep jax pmapevalstep batch replicate the train state on each device state state replicate logger info running training logger infof num examples lentraindataset logger infof num epochs numepochs logger infof instantaneous batch size per device trainingargs perdevicetrainbatchsize logger infof total train batch size w parallel distributed trainbatchsize logger infof total optimization steps totaltrainsteps traintime 0 create sampling rng rng inputrng jax random splitrng epochs tqdmrangenumepochs descfepoch 1numepochs position0 for epoch in epochs training trainstart time time create sampling rng rng inputrng jax random splitrng trainmetrics stepsperepoch lentraindataset trainbatchsize trainstepprogressbar tqdmtotalstepsperepoch desctraining position1 leavefalse train for batch in trainloader batch shardbatch state trainmetric ptrainstepstate batch trainmetrics appendtrainmetric trainstepprogressbar update1 traintime time time trainstart trainmetric unreplicatetrainmetric trainstepprogressbar close epochs write fepoch epoch 1numepochs loss trainmetric loss learning rate f trainmetric learningrate evaluating evalmetrics evalsteps lenevaldataset evalbatchsize evalstepprogressbar tqdmtotalevalsteps descevaluating position2 leavefalse for batch in evalloader model forward batch shardbatch metrics pevalstepstate params batch evalmetrics appendmetrics evalstepprogressbar update1 normalize eval metrics evalmetrics getmetricsevalmetrics evalmetrics jax treeutil treemapjnp mean evalmetrics print metrics and update progress bar evalstepprogressbar close desc fepoch epoch 1numepochs eval loss evalmetrics loss epochs writedesc epochs desc desc save metrics if hastensorboard and jax processindex 0 curstep epoch lentraindataset trainbatchsize writemetricsummarywriter trainmetrics evalmetrics traintime curstep save checkpoint after each epoch and push checkpoint to the hub if jax processindex 0 params jax devicegetunreplicatestate params model savepretrained trainingargs outputdir paramsparams pushtohubtrainingargs pushtohub commitmessagefsaving weights and logs of epoch epoch1 if name main main usr bin env python coding utf 8 2021 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license training a clip like dual encoder models using text and vision encoders in the library the script can be used to train clip like models for languages other than english by using a text encoder pre trained in the desired language currently this script support the following vision and text models vision models vit https huggingface co models filter vit clip https huggingface co models filter clip text models bert roberta https huggingface co models filter fill mask cache the result arguments pertaining to which model config tokenizer we are going to fine tune or train from scratch arguments pertaining to what data we are going to input our model for training and eval we use torchvision for faster image pre processing we need to ensure faster processing speed as it can become a bottleneck on tpu dtaset for loading image text data for tasks like clip training image captioning args root string the root path where the dataset is stored file_path string path to the file containing the image_paths and associated captions the expected format is jsonlines where each line is a json object containing to keys image_path the path to the image captions an array of captions transform callable optional a function transform that takes in an pil image and returns a transformed version e g transforms totensor target_transform callable optional a function transform that takes in the target and transforms it transforms callable optional a function transform that takes input sample and its target as entry and returns a transformed version returns a linear warmup linear_decay learning rate function if we pass only one argument to the script and it s the path to a json file let s parse it to get our arguments make one log on every process with the configuration for debugging setup logging we only want one process per machine to log things on the screen set the verbosity to info of the transformers logger on main process only set seed for torch dataloaders initialize torchvision transforms and jit them for faster processing initialize the image text dataset store some constant use collate function to tokenizer the text and convert the processed images to numpy create data loaders enable tensorboard only on the master node initialize our training create learning rate schedule create adam optimizer setup train state define gradient update step fn define eval fn summarize metrics create parallel version of the train and eval step replicate the train state on each device create sampling rng training create sampling rng train evaluating model forward normalize eval metrics print metrics and update progress bar save metrics save checkpoint after each epoch and push checkpoint to the hub
import json
import logging
import os
import sys
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Callable, Optional

import jax
import jax.numpy as jnp
import optax
import torch
from flax import jax_utils
from flax.jax_utils import unreplicate
from flax.training import train_state
from flax.training.common_utils import get_metrics, shard, shard_prng_key
from modeling_hybrid_clip import FlaxHybridCLIP
from torchvision.datasets import VisionDataset
from torchvision.io import ImageReadMode, read_image
from torchvision.transforms import CenterCrop, ConvertImageDtype, Normalize, Resize
from torchvision.transforms.functional import InterpolationMode
from tqdm import tqdm

import transformers
from transformers import AutoTokenizer, HfArgumentParser, TrainingArguments, is_tensorboard_available, set_seed


logger = logging.getLogger(__name__)


has_tensorboard = is_tensorboard_available()
if has_tensorboard:
    try:
        from flax.metrics.tensorboard import SummaryWriter
    except ImportError as ie:
        has_tensorboard = False
        print(f"Unable to display metrics through TensorBoard because some package are not installed: {ie}")

else:
    print(
        "Unable to display metrics through TensorBoard because the package is not installed: "
        "Please run pip install tensorboard to enable."
    )


@dataclass
class ModelArguments:
    

    text_model_name_or_path: str = field(
        metadata={
            "help": (
                "The text model checkpoint for weights initialization. "
                "Don't set if you want to train a model from scratch."
            )
        },
    )
    vision_model_name_or_path: str = field(
        metadata={
            "help": (
                "The vision model checkpoint for weights initialization. "
                "Don't set if you want to train a model from scratch."
            )
        },
    )
    from_pt: bool = field(
        default=True,
        metadata={"help": "whether to load the text and vision model using PyTorch checkpoints."},
    )
    config_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
    )
    tokenizer_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    use_fast_tokenizer: bool = field(
        default=True,
        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
    )
    dtype: Optional[str] = field(
        default="float32",
        metadata={
            "help": (
                "Floating-point format in which the model weights should be initialized and trained. Choose one of"
                " `[float32, float16, bfloat16]`."
            )
        },
    )


@dataclass
class DataTrainingArguments:
    

    data_dir: Optional[str] = field(default=None, metadata={"help": "The data directory containing input files."})
    train_file: Optional[str] = field(
        default=None, metadata={"help": "The input training data file (a jsonlines file)."}
    )
    validation_file: Optional[str] = field(
        default=None,
        metadata={"help": "An optional input evaluation data file (a jsonlines file)."},
    )
    max_seq_length: Optional[int] = field(
        default=72,
        metadata={
            "help": (
                "The maximum total input sequence length after tokenization. Sequences longer "
                "than this will be truncated, sequences shorter will be padded."
            )
        },
    )
    max_train_samples: Optional[int] = field(
        default=None,
        metadata={
            "help": (
                "For debugging purposes or quicker training, truncate the number of training examples to this "
                "value if set."
            )
        },
    )
    max_eval_samples: Optional[int] = field(
        default=None,
        metadata={
            "help": (
                "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
                "value if set."
            )
        },
    )
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
    )
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
    )
    preprocessing_num_workers: Optional[int] = field(
        default=None,
        metadata={"help": "The number of processes to use for the preprocessing."},
    )

    def __post_init__(self):
        if self.train_file is None and self.validation_file is None:
            raise ValueError("Need either a dataset name or a training/validation file.")
        else:
            if self.train_file is not None:
                extension = self.train_file.split(".")[-1]
                assert extension == "json", "`train_file` should be a json file."
            if self.validation_file is not None:
                extension = self.validation_file.split(".")[-1]
                assert extension == "json", "`validation_file` should be a json file."




class Transform(torch.nn.Module):
    def __init__(self, image_size):
        super().__init__()
        self.transforms = torch.nn.Sequential(
            Resize([image_size], interpolation=InterpolationMode.BICUBIC),
            CenterCrop(image_size),
            ConvertImageDtype(torch.float),
            Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        with torch.no_grad():
            x = self.transforms(x)
        return x


class ImageTextDataset(VisionDataset):
    

    def __init__(
        self,
        root: str,
        file_path: str,
        captions_per_image=2,
        transform: Optional[Callable] = None,
        target_transform: Optional[Callable] = None,
        transforms: Optional[Callable] = None,
    ):
        super().__init__(root, transforms, transform, target_transform)

        with open(file_path, "r") as f:
            examples = [json.loads(line) for line in f.readlines()]

        self.captions = []
        self.image_paths = []

        for example in examples:
            captions_subset = example["captions"][:captions_per_image]
            self.captions.extend(captions_subset)
            self.image_paths.extend([example["image_path"]] * len(captions_subset))

    def _load_image(self, idx: int):
        path = self.image_paths[idx]
        return read_image(path, mode=ImageReadMode.RGB)

    def _load_target(self, idx):
        return self.captions[idx]

    def __getitem__(self, index: int):
        image = self._load_image(index)
        target = self._load_target(index)

        if self.transforms is not None:
            image, target = self.transforms(image, target)

        return image, target

    def __len__(self) -> int:
        return len(self.captions)


class TrainState(train_state.TrainState):
    dropout_rng: jnp.ndarray

    def replicate(self):
        return jax_utils.replicate(self).replace(dropout_rng=shard_prng_key(self.dropout_rng))


def write_metric(summary_writer, train_metrics, eval_metrics, train_time, step):
    summary_writer.scalar("train_time", train_time, step)

    train_metrics = get_metrics(train_metrics)
    for key, vals in train_metrics.items():
        tag = f"train_{key}"
        for i, val in enumerate(vals):
            summary_writer.scalar(tag, val, step - len(vals) + i + 1)

    for metric_name, value in eval_metrics.items():
        summary_writer.scalar(f"eval_{metric_name}", value, step)


def create_learning_rate_fn(
    train_ds_size: int, train_batch_size: int, num_train_epochs: int, num_warmup_steps: int, learning_rate: float
) -> Callable[[int], jnp.ndarray]:
    
    steps_per_epoch = train_ds_size // train_batch_size
    num_train_steps = steps_per_epoch * num_train_epochs
    warmup_fn = optax.linear_schedule(init_value=0.0, end_value=learning_rate, transition_steps=num_warmup_steps)
    decay_fn = optax.linear_schedule(
        init_value=learning_rate, end_value=0, transition_steps=num_train_steps - num_warmup_steps
    )
    schedule_fn = optax.join_schedules(schedules=[warmup_fn, decay_fn], boundaries=[num_warmup_steps])
    return schedule_fn


def main():
    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
        
        
        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
    else:
        model_args, data_args, training_args = parser.parse_args_into_dataclasses()

    if (
        os.path.exists(training_args.output_dir)
        and os.listdir(training_args.output_dir)
        and training_args.do_train
        and not training_args.overwrite_output_dir
    ):
        raise ValueError(
            f"Output directory ({training_args.output_dir}) already exists and is not empty. "
            "Use --overwrite_output_dir to overcome."
        )

    
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        level=logging.INFO,
    )
    
    logger.setLevel(logging.INFO if jax.process_index() == 0 else logging.ERROR)
    if jax.process_index() == 0:
        transformers.utils.logging.set_verbosity_info()
    else:
        transformers.utils.logging.set_verbosity_error()

    
    logger.info(f"Training/evaluation parameters {training_args}")

    if model_args.tokenizer_name:
        tokenizer = AutoTokenizer.from_pretrained(
            model_args.tokenizer_name, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer
        )
    elif model_args.text_model_name_or_path:
        tokenizer = AutoTokenizer.from_pretrained(
            model_args.text_model_name_or_path, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer
        )
    else:
        raise ValueError(
            "You are instantiating a new tokenizer from scratch. This is not supported by this script. "
            "You can do it from another script, save it, and load it from here, using --tokenizer_name."
        )

    model = FlaxHybridCLIP.from_text_vision_pretrained(
        model_args.text_model_name_or_path,
        model_args.vision_model_name_or_path,
        seed=training_args.seed,
        dtype=getattr(jnp, model_args.dtype),
        text_from_pt=model_args.from_pt,
        vision_from_pt=model_args.from_pt,
    )
    config = model.config
    
    set_seed(training_args.seed)

    
    preprocess = Transform(config.vision_config.image_size)
    preprocess = torch.jit.script(preprocess)

    
    train_dataset = ImageTextDataset(
        data_args.data_dir,
        data_args.train_file,
        captions_per_image=2,
        transform=preprocess,
    )

    eval_dataset = ImageTextDataset(
        data_args.data_dir,
        data_args.validation_file,
        captions_per_image=1,
        transform=preprocess,
    )

    
    num_epochs = int(training_args.num_train_epochs)
    train_batch_size = int(training_args.per_device_train_batch_size) * jax.device_count()
    eval_batch_size = int(training_args.per_device_eval_batch_size) * jax.device_count()
    steps_per_epoch = len(train_dataset) // train_batch_size
    total_train_steps = steps_per_epoch * num_epochs

    
    def collate_fn(examples):
        pixel_values = torch.stack([example[0] for example in examples]).permute(0, 2, 3, 1).numpy()
        captions = [example[1] for example in examples]
        inputs = tokenizer(
            captions, max_length=data_args.max_seq_length, padding="max_length", truncation=True, return_tensors="np"
        )

        batch = {
            "pixel_values": pixel_values,
            "input_ids": inputs["input_ids"],
            "attention_mask": inputs["attention_mask"],
        }

        return batch

    
    train_loader = torch.utils.data.DataLoader(
        train_dataset,
        batch_size=train_batch_size,
        shuffle=True,
        num_workers=data_args.preprocessing_num_workers,
        persistent_workers=True,
        drop_last=True,
        collate_fn=collate_fn,
    )

    eval_loader = torch.utils.data.DataLoader(
        eval_dataset,
        batch_size=eval_batch_size,
        shuffle=False,
        num_workers=data_args.preprocessing_num_workers,
        persistent_workers=True,
        drop_last=True,
        collate_fn=collate_fn,
    )

    
    if has_tensorboard and jax.process_index() == 0:
        summary_writer = SummaryWriter(log_dir=Path(training_args.output_dir).joinpath("logs").as_posix())

    
    rng = jax.random.PRNGKey(training_args.seed)
    rng, dropout_rng = jax.random.split(rng)

    
    linear_decay_lr_schedule_fn = create_learning_rate_fn(
        len(train_dataset),
        train_batch_size,
        training_args.num_train_epochs,
        training_args.warmup_steps,
        training_args.learning_rate,
    )

    
    adamw = optax.adamw(
        learning_rate=linear_decay_lr_schedule_fn,
        b1=training_args.adam_beta1,
        b2=training_args.adam_beta2,
        eps=training_args.adam_epsilon,
        weight_decay=training_args.weight_decay,
    )

    
    state = TrainState.create(apply_fn=model.__call__, params=model.params, tx=adamw, dropout_rng=dropout_rng)

    def cross_entropy(logits, axis):
        logprobs = jax.nn.log_softmax(logits, axis=axis)
        nll = jnp.diag(logprobs)
        ce = -jnp.mean(nll)
        return ce

    def clip_loss(similarity):
        loss = (cross_entropy(similarity, axis=0) + cross_entropy(similarity, axis=1)) / 2
        return loss

    
    def train_step(state, batch):
        dropout_rng, new_dropout_rng = jax.random.split(state.dropout_rng)

        def compute_loss(params):
            logits = state.apply_fn(**batch, params=params, dropout_rng=dropout_rng, train=True)[0]
            loss = clip_loss(logits)
            return loss

        grad_fn = jax.value_and_grad(compute_loss)
        loss, grad = grad_fn(state.params)
        grad = jax.lax.pmean(grad, "batch")

        new_state = state.apply_gradients(grads=grad, dropout_rng=new_dropout_rng)

        metrics = {"loss": loss, "learning_rate": linear_decay_lr_schedule_fn(state.step)}
        metrics = jax.lax.pmean(metrics, axis_name="batch")

        return new_state, metrics

    
    def eval_step(params, batch):
        logits = model(**batch, params=params, train=False)[0]
        loss = clip_loss(logits)

        
        metrics = {"loss": loss}
        metrics = jax.lax.pmean(metrics, axis_name="batch")
        return metrics

    
    p_train_step = jax.pmap(train_step, "batch", donate_argnums=(0,))
    p_eval_step = jax.pmap(eval_step, "batch")

    
    state = state.replicate()

    logger.info("***** Running training *****")
    logger.info(f"  Num examples = {len(train_dataset)}")
    logger.info(f"  Num Epochs = {num_epochs}")
    logger.info(f"  Instantaneous batch size per device = {training_args.per_device_train_batch_size}")
    logger.info(f"  Total train batch size (w. parallel & distributed) = {train_batch_size}")
    logger.info(f"  Total optimization steps = {total_train_steps}")

    train_time = 0
    
    rng, input_rng = jax.random.split(rng)

    epochs = tqdm(range(num_epochs), desc=f"Epoch ... (1/{num_epochs})", position=0)
    for epoch in epochs:
        
        train_start = time.time()

        
        rng, input_rng = jax.random.split(rng)
        train_metrics = []

        steps_per_epoch = len(train_dataset) // train_batch_size
        train_step_progress_bar = tqdm(total=steps_per_epoch, desc="Training...", position=1, leave=False)
        
        for batch in train_loader:
            batch = shard(batch)
            state, train_metric = p_train_step(state, batch)
            train_metrics.append(train_metric)

            train_step_progress_bar.update(1)

        train_time += time.time() - train_start

        train_metric = unreplicate(train_metric)

        train_step_progress_bar.close()
        epochs.write(
            f"Epoch... ({epoch + 1}/{num_epochs} | Loss: {train_metric['loss']}, Learning Rate:"
            f" {train_metric['learning_rate']})"
        )

        
        eval_metrics = []
        eval_steps = len(eval_dataset) // eval_batch_size
        eval_step_progress_bar = tqdm(total=eval_steps, desc="Evaluating...", position=2, leave=False)
        for batch in eval_loader:
            
            batch = shard(batch)
            metrics = p_eval_step(state.params, batch)
            eval_metrics.append(metrics)

            eval_step_progress_bar.update(1)

        
        eval_metrics = get_metrics(eval_metrics)

        eval_metrics = jax.tree_util.tree_map(jnp.mean, eval_metrics)

        
        eval_step_progress_bar.close()
        desc = f"Epoch... ({epoch + 1}/{num_epochs} | Eval Loss: {eval_metrics['loss']})"
        epochs.write(desc)
        epochs.desc = desc

        
        if has_tensorboard and jax.process_index() == 0:
            cur_step = epoch * (len(train_dataset) // train_batch_size)
            write_metric(summary_writer, train_metrics, eval_metrics, train_time, cur_step)

        
        if jax.process_index() == 0:
            params = jax.device_get(unreplicate(state.params))
            model.save_pretrained(
                training_args.output_dir,
                params=params,
                push_to_hub=training_args.push_to_hub,
                commit_message=f"Saving weights and logs of epoch {epoch+1}",
            )


if __name__ == "__main__":
    main()