




import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to properly calculate the metrics on the
# validation dataset when in a distributed system, and builds off the
# `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


_lowerCAmelCase  :      Any						=      1_6
_lowerCAmelCase  :      str						=      3_2



def   lowerCAmelCase						(						_lowerCAmelCase							:      Accelerator  ,							_lowerCAmelCase							:      int = 16		):




 """simple docstring"""







 UpperCAmelCase__							       =  AutoTokenizer.from_pretrained("bert-base-cased"		)
 UpperCAmelCase__							       =  load_dataset("glue"  ,							"mrpc"		)

 def tokenize_function(_lowerCAmelCase							:      List[Any]		):
  # max_length=None => use the model max length (it's actually the default)
  UpperCAmelCase__							       =  tokenizer(examples["sentence1"]  ,							examples["sentence2"]  ,							truncation=_lowerCAmelCase  ,							max_length=_lowerCAmelCase		)
  return outputs

 # Apply the method we just defined to all the examples in all the splits of the dataset
 # starting with the main process first:
 with accelerator.main_process_first():
  UpperCAmelCase__							       =  datasets.map(
      _lowerCAmelCase  ,							batched=_lowerCAmelCase  ,							remove_columns=["idx", "sentence1", "sentence2"]  ,							)

 # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
 # transformers library
 UpperCAmelCase__							       =  tokenized_datasets.rename_column("label"  ,							"labels"		)

 def collate_fn(_lowerCAmelCase							:      str		):
  # On TPU it's best to pad everything to the same length or training will be very slow.
  UpperCAmelCase__							       =  128 if accelerator.distributed_type == DistributedType.TPU else None
  # When using mixed precision we want round multiples of 8/16
  if accelerator.mixed_precision == "fp8":
   UpperCAmelCase__							       =  16
  elif accelerator.mixed_precision != "no":
   UpperCAmelCase__							       =  8
  else:
   UpperCAmelCase__							       =  None

  return tokenizer.pad(
      _lowerCAmelCase  ,							padding="longest"  ,							max_length=_lowerCAmelCase  ,							pad_to_multiple_of=_lowerCAmelCase  ,							return_tensors="pt"  ,							)

 # Instantiate dataloaders.
 UpperCAmelCase__							       =  DataLoader(
     tokenized_datasets["train"]  ,							shuffle=_lowerCAmelCase  ,							collate_fn=_lowerCAmelCase  ,							batch_size=_lowerCAmelCase		)
 UpperCAmelCase__							       =  DataLoader(
     tokenized_datasets["validation"]  ,							shuffle=_lowerCAmelCase  ,							collate_fn=_lowerCAmelCase  ,							batch_size=_lowerCAmelCase		)

 return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
  from accelerate.test_utils.training import mocked_dataloaders

  _lowerCAmelCase  :      str						=      mocked_dataloaders  # noqa: F811



def   lowerCAmelCase						(						_lowerCAmelCase							:      List[Any]  ,							_lowerCAmelCase							:      str		):




 """simple docstring"""







 if os.environ.get("TESTING_MOCKED_DATALOADERS"  ,							_lowerCAmelCase		) == "1":
  UpperCAmelCase__							       =  2
 # Initialize accelerator
 UpperCAmelCase__							       =  Accelerator(cpu=args.cpu  ,							mixed_precision=args.mixed_precision		)
 # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
 UpperCAmelCase__							       =  config["lr"]
 UpperCAmelCase__							       =  int(config["num_epochs"]		)
 UpperCAmelCase__							       =  int(config["seed"]		)
 UpperCAmelCase__							       =  int(config["batch_size"]		)

 UpperCAmelCase__							       =  evaluate.load("glue"  ,							"mrpc"		)

 # If the batch size is too big we use gradient accumulation
 UpperCAmelCase__							       =  1
 if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
  UpperCAmelCase__							       =  batch_size // MAX_GPU_BATCH_SIZE
  UpperCAmelCase__							       =  MAX_GPU_BATCH_SIZE

 set_seed(_lowerCAmelCase		)
 UpperCAmelCase__							       =  get_dataloaders(_lowerCAmelCase  ,							_lowerCAmelCase		)
 # Instantiate the model (we build the model here so that the seed also control new weights initialization)
 UpperCAmelCase__							       =  AutoModelForSequenceClassification.from_pretrained("bert-base-cased"  ,							return_dict=_lowerCAmelCase		)

 # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
 # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
 # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
 UpperCAmelCase__							       =  model.to(accelerator.device		)

 # Instantiate optimizer
 UpperCAmelCase__							       =  AdamW(params=model.parameters()  ,							lr=_lowerCAmelCase		)

 # Instantiate scheduler
 UpperCAmelCase__							       =  get_linear_schedule_with_warmup(
     optimizer=_lowerCAmelCase  ,							num_warmup_steps=100  ,							num_training_steps=(len(_lowerCAmelCase		) * num_epochs) // gradient_accumulation_steps  ,							)

 # Prepare everything
 # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
 # prepare method.
 UpperCAmelCase__							       =  accelerator.prepare(
     _lowerCAmelCase  ,							_lowerCAmelCase  ,							_lowerCAmelCase  ,							_lowerCAmelCase  ,							_lowerCAmelCase		)

 # Now we train the model
 for epoch in range(_lowerCAmelCase		):
  model.train()
  for step, batch in enumerate(_lowerCAmelCase		):
   # We could avoid this line since we set the accelerator with `device_placement=True`.
   batch.to(accelerator.device		)
   UpperCAmelCase__							       =  model(**_lowerCAmelCase		)
   UpperCAmelCase__							       =  outputs.loss
   UpperCAmelCase__							       =  loss / gradient_accumulation_steps
   accelerator.backward(_lowerCAmelCase		)
   if step % gradient_accumulation_steps == 0:
    optimizer.step()
    lr_scheduler.step()
    optimizer.zero_grad()

  model.eval()
  UpperCAmelCase__							       =  0
  for step, batch in enumerate(_lowerCAmelCase		):
   # We could avoid this line since we set the accelerator with `device_placement=True`.
   batch.to(accelerator.device		)
   with torch.no_grad():
    UpperCAmelCase__							       =  model(**_lowerCAmelCase		)
   UpperCAmelCase__							       =  outputs.logits.argmax(dim=-1		)
   UpperCAmelCase__							       =  accelerator.gather((predictions, batch["labels"])		)
   # New Code #
   # First we check if it's a distributed system
   if accelerator.use_distributed:
    # Then see if we're on the last batch of our eval dataloader
    if step == len(_lowerCAmelCase		) - 1:
     # Last batch needs to be truncated on distributed systems as it contains additional samples
     UpperCAmelCase__							       =  predictions[: len(eval_dataloader.dataset		) - samples_seen]
     UpperCAmelCase__							       =  references[: len(eval_dataloader.dataset		) - samples_seen]
    else:
     # Otherwise we add the number of samples seen
     samples_seen += references.shape[0]
       # All of this can be avoided if you use `Accelerator.gather_for_metrics` instead of `Accelerator.gather`:
       # accelerator.gather_for_metrics((predictions, batch["labels"]))
   metric.add_batch(
       predictions=_lowerCAmelCase  ,							references=_lowerCAmelCase  ,							)

  UpperCAmelCase__							       =  metric.compute()
  # Use accelerator.print to print only on the main process.
  accelerator.print(F'''epoch {epoch}:'''  ,							_lowerCAmelCase		)



def   lowerCAmelCase						(						):




 """simple docstring"""







 UpperCAmelCase__							       =  argparse.ArgumentParser(description="Simple example of training script."		)
 parser.add_argument(
     "--mixed_precision"  ,							type=_lowerCAmelCase  ,							default=_lowerCAmelCase  ,							choices=["no", "fp16", "bf16", "fp8"]  ,							help="Whether to use mixed precision. Choose"
     "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
     "and an Nvidia Ampere GPU."  ,							)
 parser.add_argument("--cpu"  ,							action="store_true"  ,							help="If passed, will train on the CPU."		)
 UpperCAmelCase__							       =  parser.parse_args()
 UpperCAmelCase__							       =  {"lr": 2E-5, "num_epochs": 3, "seed": 42, "batch_size": 16}
 training_function(_lowerCAmelCase  ,							_lowerCAmelCase		)


if __name__ == "__main__":
  main()












import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
     from PIL import Image

     from transformers import CLIPSegProcessor, ViTImageProcessor







@require_vision
class 				__lowerCAmelCase       (    unittest.TestCase):







      def    _lowercase  (    self							)   ->						Dict:



        '''simple docstring'''




        a__	:					Any										=tempfile.mkdtemp()

        # fmt: off
        a__	:					List[Any]										=["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
        # fmt: on
        a__	:					str										=dict(zip(lowerCAmelCase__		,      range(len(lowerCAmelCase__							)							)							)							)
        a__	:					List[Any]										=["#version: 0.2", "l o", "lo w</w>", "e r</w>", ""]
        a__	:					Optional[int]										={"unk_token": "<unk>"}

        a__	:					Optional[Any]										=os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES["vocab_file"]							)
        a__	:					Tuple										=os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES["merges_file"]							)
        with open(self.vocab_file		,      "w"		,      encoding="utf-8"							) as fp:
          fp.write(json.dumps(lowerCAmelCase__							) + "\n"							)
        with open(self.merges_file		,      "w"		,      encoding="utf-8"							) as fp:
          fp.write("\n".join(lowerCAmelCase__							)							)

        a__	:					Optional[Any]										={
            "do_resize": True,
            "size": 2_0,
            "do_center_crop": True,
            "crop_size": 1_8,
            "do_normalize": True,
            "image_mean": [0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73],
            "image_std": [0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11],
        }
        a__	:					Dict										=os.path.join(self.tmpdirname		,      lowerCAmelCase__							)
        with open(self.image_processor_file		,      "w"		,      encoding="utf-8"							) as fp:
          json.dump(lowerCAmelCase__		,      lowerCAmelCase__							)







      def    _lowercase  (    self		,      **lowerCAmelCase__							)   ->						Union[str, Any]:



        '''simple docstring'''




        return CLIPTokenizer.from_pretrained(self.tmpdirname		,      **lowerCAmelCase__							)







      def    _lowercase  (    self		,      **lowerCAmelCase__							)   ->						List[Any]:



        '''simple docstring'''




        return CLIPTokenizerFast.from_pretrained(self.tmpdirname		,      **lowerCAmelCase__							)







      def    _lowercase  (    self		,      **lowerCAmelCase__							)   ->						Any:



        '''simple docstring'''




        return ViTImageProcessor.from_pretrained(self.tmpdirname		,      **lowerCAmelCase__							)







      def    _lowercase  (    self							)   ->						Optional[int]:



        '''simple docstring'''




        shutil.rmtree(self.tmpdirname							)







      def    _lowercase  (    self							)   ->						Any:



        '''simple docstring'''




        a__	:					Optional[Any]										=[np.random.randint(2_5_5		,      size=(3, 3_0, 4_0_0)		,      dtype=np.uinta							)]

        a__	:					List[Any]										=[Image.fromarray(np.moveaxis(lowerCAmelCase__		,      0		,      -1							)							) for x in image_inputs]

        return image_inputs







      def    _lowercase  (    self							)   ->						Dict:



        '''simple docstring'''




        a__	:					Union[str, Any]										=self.get_tokenizer()
        a__	:					int										=self.get_rust_tokenizer()
        a__	:					List[str]										=self.get_image_processor()

        a__	:					Dict										=CLIPSegProcessor(tokenizer=lowerCAmelCase__		,      image_processor=lowerCAmelCase__							)
        processor_slow.save_pretrained(self.tmpdirname							)
        a__	:					Optional[Any]										=CLIPSegProcessor.from_pretrained(self.tmpdirname		,      use_fast=lowerCAmelCase__							)

        a__	:					Tuple										=CLIPSegProcessor(tokenizer=lowerCAmelCase__		,      image_processor=lowerCAmelCase__							)
        processor_fast.save_pretrained(self.tmpdirname							)
        a__	:					Dict										=CLIPSegProcessor.from_pretrained(self.tmpdirname							)

        self.assertEqual(processor_slow.tokenizer.get_vocab()		,      tokenizer_slow.get_vocab()							)
        self.assertEqual(processor_fast.tokenizer.get_vocab()		,      tokenizer_fast.get_vocab()							)
        self.assertEqual(tokenizer_slow.get_vocab()		,      tokenizer_fast.get_vocab()							)
        self.assertIsInstance(processor_slow.tokenizer		,      lowerCAmelCase__							)
        self.assertIsInstance(processor_fast.tokenizer		,      lowerCAmelCase__							)

        self.assertEqual(processor_slow.image_processor.to_json_string()		,      image_processor.to_json_string()							)
        self.assertEqual(processor_fast.image_processor.to_json_string()		,      image_processor.to_json_string()							)
        self.assertIsInstance(processor_slow.image_processor		,      lowerCAmelCase__							)
        self.assertIsInstance(processor_fast.image_processor		,      lowerCAmelCase__							)







      def    _lowercase  (    self							)   ->						Any:



        '''simple docstring'''




        a__	:					List[str]										=CLIPSegProcessor(tokenizer=self.get_tokenizer()		,      image_processor=self.get_image_processor()							)
        processor.save_pretrained(self.tmpdirname							)

        a__	:					str										=self.get_tokenizer(bos_token="(BOS)"		,      eos_token="(EOS)"							)
        a__	:					int										=self.get_image_processor(do_normalize=lowerCAmelCase__		,      padding_value=1.0							)

        a__	:					Optional[Any]										=CLIPSegProcessor.from_pretrained(
            self.tmpdirname		,      bos_token="(BOS)"		,      eos_token="(EOS)"		,      do_normalize=lowerCAmelCase__		,      padding_value=1.0							)

        self.assertEqual(processor.tokenizer.get_vocab()		,      tokenizer_add_kwargs.get_vocab()							)
        self.assertIsInstance(processor.tokenizer		,      lowerCAmelCase__							)

        self.assertEqual(processor.image_processor.to_json_string()		,      image_processor_add_kwargs.to_json_string()							)
        self.assertIsInstance(processor.image_processor		,      lowerCAmelCase__							)







      def    _lowercase  (    self							)   ->						List[Any]:



        '''simple docstring'''




        a__	:					str										=self.get_image_processor()
        a__	:					Optional[int]										=self.get_tokenizer()

        a__	:					Dict										=CLIPSegProcessor(tokenizer=lowerCAmelCase__		,      image_processor=lowerCAmelCase__							)

        a__	:					str										=self.prepare_image_inputs()

        a__	:					Any										=image_processor(lowerCAmelCase__		,      return_tensors="np"							)
        a__	:					Optional[int]										=processor(images=lowerCAmelCase__		,      return_tensors="np"							)

        for key in input_feat_extract.keys():
          self.assertAlmostEqual(input_feat_extract[key].sum()		,      input_processor[key].sum()		,      delta=1E-2							)







      def    _lowercase  (    self							)   ->						Union[str, Any]:



        '''simple docstring'''




        a__	:					Optional[int]										=self.get_image_processor()
        a__	:					List[Any]										=self.get_tokenizer()

        a__	:					Optional[int]										=CLIPSegProcessor(tokenizer=lowerCAmelCase__		,      image_processor=lowerCAmelCase__							)

        a__	:					Union[str, Any]										="lower newer"

        a__	:					List[str]										=processor(text=lowerCAmelCase__							)

        a__	:					str										=tokenizer(lowerCAmelCase__							)

        for key in encoded_tok.keys():
          self.assertListEqual(encoded_tok[key]		,      encoded_processor[key]							)







      def    _lowercase  (    self							)   ->						Optional[int]:



        '''simple docstring'''




        a__	:					Any										=self.get_image_processor()
        a__	:					Dict										=self.get_tokenizer()

        a__	:					Union[str, Any]										=CLIPSegProcessor(tokenizer=lowerCAmelCase__		,      image_processor=lowerCAmelCase__							)

        a__	:					Dict										="lower newer"
        a__	:					int										=self.prepare_image_inputs()

        a__	:					Any										=processor(text=lowerCAmelCase__		,      images=lowerCAmelCase__							)

        self.assertListEqual(list(inputs.keys()							)		,      ["input_ids", "attention_mask", "pixel_values"]							)

        # test if it raises when no input is passed
        with pytest.raises(lowerCAmelCase__							):
          processor()







      def    _lowercase  (    self							)   ->						str:



        '''simple docstring'''




        a__	:					Union[str, Any]										=self.get_image_processor()
        a__	:					Optional[Any]										=self.get_tokenizer()

        a__	:					str										=CLIPSegProcessor(tokenizer=lowerCAmelCase__		,      image_processor=lowerCAmelCase__							)

        a__	:					int										=self.prepare_image_inputs()
        a__	:					Union[str, Any]										=self.prepare_image_inputs()

        a__	:					Tuple										=processor(images=lowerCAmelCase__		,      visual_prompt=lowerCAmelCase__							)

        self.assertListEqual(list(inputs.keys()							)		,      ["pixel_values", "conditional_pixel_values"]							)

        # test if it raises when no input is passed
        with pytest.raises(lowerCAmelCase__							):
          processor()







      def    _lowercase  (    self							)   ->						Tuple:



        '''simple docstring'''




        a__	:					Optional[int]										=self.get_image_processor()
        a__	:					Any										=self.get_tokenizer()

        a__	:					Tuple										=CLIPSegProcessor(tokenizer=lowerCAmelCase__		,      image_processor=lowerCAmelCase__							)

        a__	:					Dict										=[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

        a__	:					Optional[Any]										=processor.batch_decode(lowerCAmelCase__							)
        a__	:					Dict										=tokenizer.batch_decode(lowerCAmelCase__							)

        self.assertListEqual(lowerCAmelCase__		,      lowerCAmelCase__							)






