


import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers.testing_utils import require_vision
from transformers.utils import is_vision_available


if is_vision_available():
		from PIL import Image

		from transformers import AutoProcessor, BertTokenizer, BlipImageProcessor, BlipProcessor, PreTrainedTokenizerFast



@require_vision
class 						_UpperCAmelCase      (     unittest.TestCase							):






		def _snake_case		(						self				:     Any):
				SCREAMING_SNAKE_CASE_				:Dict				     =						tempfile.mkdtemp()

				SCREAMING_SNAKE_CASE_				:List[str]				     =						BlipImageProcessor()
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-BertModel")

				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						BlipProcessor(UpperCAmelCase   ,  UpperCAmelCase)

				processor.save_pretrained(self.tmpdirname)






		def _snake_case		(						self				:     List[str]   ,  **UpperCAmelCase				:     str):
				return AutoProcessor.from_pretrained(self.tmpdirname   ,  **UpperCAmelCase).tokenizer






		def _snake_case		(						self				:     int   ,  **UpperCAmelCase				:     int):
				return AutoProcessor.from_pretrained(self.tmpdirname   ,  **UpperCAmelCase).image_processor






		def _snake_case		(						self				:     Optional[Any]):
				shutil.rmtree(self.tmpdirname)






		def _snake_case		(						self				:     Dict):
				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						[np.random.randint(2_55   ,  size=(3, 30, 4_00)   ,  dtype=np.uinta)]

				SCREAMING_SNAKE_CASE_				:List[str]				     =						[Image.fromarray(np.moveaxis(UpperCAmelCase   ,  0   ,  -1)) for x in image_inputs]

				return image_inputs






		def _snake_case		(						self				:     List[str]):
				SCREAMING_SNAKE_CASE_				:int				     =						BlipProcessor(tokenizer=self.get_tokenizer()   ,  image_processor=self.get_image_processor())
				processor.save_pretrained(self.tmpdirname)

				SCREAMING_SNAKE_CASE_				:List[str]				     =						self.get_tokenizer(bos_token="(BOS)"   ,  eos_token="(EOS)")
				SCREAMING_SNAKE_CASE_				:int				     =						self.get_image_processor(do_normalize=UpperCAmelCase   ,  padding_value=1.0)

				SCREAMING_SNAKE_CASE_				:Dict				     =						BlipProcessor.from_pretrained(
				    self.tmpdirname   ,  bos_token="(BOS)"   ,  eos_token="(EOS)"   ,  do_normalize=UpperCAmelCase   ,  padding_value=1.0)

				self.assertEqual(processor.tokenizer.get_vocab()   ,  tokenizer_add_kwargs.get_vocab())
				self.assertIsInstance(processor.tokenizer   ,  UpperCAmelCase)

				self.assertEqual(processor.image_processor.to_json_string()   ,  image_processor_add_kwargs.to_json_string())
				self.assertIsInstance(processor.image_processor   ,  UpperCAmelCase)






		def _snake_case		(						self				:     Dict):
				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						self.get_image_processor()
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						self.get_tokenizer()

				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						BlipProcessor(tokenizer=UpperCAmelCase   ,  image_processor=UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:Any				     =						self.prepare_image_inputs()

				SCREAMING_SNAKE_CASE_				:List[Any]				     =						image_processor(UpperCAmelCase   ,  return_tensors="np")
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						processor(images=UpperCAmelCase   ,  return_tensors="np")

				for key in input_feat_extract.keys():
						self.assertAlmostEqual(input_feat_extract[key].sum()   ,  input_processor[key].sum()   ,  delta=1E-2)






		def _snake_case		(						self				:     Union[str, Any]):
				SCREAMING_SNAKE_CASE_				:List[str]				     =						self.get_image_processor()
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						self.get_tokenizer()

				SCREAMING_SNAKE_CASE_				:Any				     =						BlipProcessor(tokenizer=UpperCAmelCase   ,  image_processor=UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:Dict				     =						"lower newer"

				SCREAMING_SNAKE_CASE_				:str				     =						processor(text=UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:Any				     =						tokenizer(UpperCAmelCase   ,  return_token_type_ids=UpperCAmelCase)

				for key in encoded_tok.keys():
						self.assertListEqual(encoded_tok[key]   ,  encoded_processor[key])






		def _snake_case		(						self				:     Union[str, Any]):
				SCREAMING_SNAKE_CASE_				:Tuple				     =						self.get_image_processor()
				SCREAMING_SNAKE_CASE_				:List[str]				     =						self.get_tokenizer()

				SCREAMING_SNAKE_CASE_				:Dict				     =						BlipProcessor(tokenizer=UpperCAmelCase   ,  image_processor=UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						"lower newer"
				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						self.prepare_image_inputs()

				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						processor(text=UpperCAmelCase   ,  images=UpperCAmelCase)

				self.assertListEqual(list(inputs.keys())   ,  ["pixel_values", "input_ids", "attention_mask"])

				# test if it raises when no input is passed
				with pytest.raises(UpperCAmelCase):
						processor()






		def _snake_case		(						self				:     Union[str, Any]):
				SCREAMING_SNAKE_CASE_				:Tuple				     =						self.get_image_processor()
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						self.get_tokenizer()

				SCREAMING_SNAKE_CASE_				:Dict				     =						BlipProcessor(tokenizer=UpperCAmelCase   ,  image_processor=UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:List[str]				     =						[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

				SCREAMING_SNAKE_CASE_				:Any				     =						processor.batch_decode(UpperCAmelCase)
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						tokenizer.batch_decode(UpperCAmelCase)

				self.assertListEqual(UpperCAmelCase   ,  UpperCAmelCase)







		def _snake_case		(						self				:     int):
				SCREAMING_SNAKE_CASE_				:Dict				     =						self.get_image_processor()
				SCREAMING_SNAKE_CASE_				:Dict				     =						self.get_tokenizer()

				SCREAMING_SNAKE_CASE_				:Dict				     =						BlipProcessor(tokenizer=UpperCAmelCase   ,  image_processor=UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:Tuple				     =						"lower newer"
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						self.prepare_image_inputs()

				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						processor(text=UpperCAmelCase   ,  images=UpperCAmelCase)

				# For now the processor supports only ['pixel_values', 'input_ids', 'attention_mask']
				self.assertListEqual(list(inputs.keys())   ,  ["pixel_values", "input_ids", "attention_mask"])










import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version(">=", FSDP_PYTORCH_VERSION):
		import torch.distributed.checkpoint as dist_cp
		from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
		from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
		from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
		from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


SCREAMING_SNAKE_CASE__     					=			get_logger(__name__)


def     lowercase	(    a     , a     , a     , a     , a=0 ):



		'''simple docstring'''




		os.makedirs(a     , exist_ok=a )
		with FSDP.state_dict_type(
		    a     , fsdp_plugin.state_dict_type     , fsdp_plugin.state_dict_config     , fsdp_plugin.optim_state_dict_config ):
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						model.state_dict()
				if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
						SCREAMING_SNAKE_CASE_				:Optional[int]				     =						F"{MODEL_NAME}.bin" if model_index == 0 else F"{MODEL_NAME}_{model_index}.bin"
						SCREAMING_SNAKE_CASE_				:str				     =						os.path.join(a     , a )
						if accelerator.process_index == 0:
								logger.info(F"Saving model to {output_model_file}" )
								torch.save(a     , a )
								logger.info(F"Model saved to {output_model_file}" )
				elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
						SCREAMING_SNAKE_CASE_				:Tuple				     =						(
						    F"{MODEL_NAME}_rank{accelerator.process_index}.bin"
						    if model_index == 0
						    else F"{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"
						)
						SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						os.path.join(a     , a )
						logger.info(F"Saving model to {output_model_file}" )
						torch.save(a     , a )
						logger.info(F"Model saved to {output_model_file}" )
				elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
						SCREAMING_SNAKE_CASE_				:Dict				     =						os.path.join(a     , F"{MODEL_NAME}_{model_index}" )
						os.makedirs(a     , exist_ok=a )
						logger.info(F"Saving model to {ckpt_dir}" )
						SCREAMING_SNAKE_CASE_				:int				     =						{"model": state_dict}

						dist_cp.save_state_dict(
						    state_dict=a     , storage_writer=dist_cp.FileSystemWriter(a )     , planner=DefaultSavePlanner()     , )
						logger.info(F"Model saved to {ckpt_dir}" )


def     lowercase	(    a     , a     , a     , a     , a=0 ):



		'''simple docstring'''




		accelerator.wait_for_everyone()
		with FSDP.state_dict_type(
		    a     , fsdp_plugin.state_dict_type     , fsdp_plugin.state_dict_config     , fsdp_plugin.optim_state_dict_config ):
				if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
						if type(a ) != FSDP and accelerator.process_index != 0:
								if not fsdp_plugin.sync_module_states:
										raise ValueError(
										    "Set the `sync_module_states` flag to `True` so that model states are synced across processes when "
										    "initializing FSDP object" )
								return
						SCREAMING_SNAKE_CASE_				:List[Any]				     =						F"{MODEL_NAME}.bin" if model_index == 0 else F"{MODEL_NAME}_{model_index}.bin"
						SCREAMING_SNAKE_CASE_				:int				     =						os.path.join(a     , a )
						logger.info(F"Loading model from {input_model_file}" )
						SCREAMING_SNAKE_CASE_				:Dict				     =						torch.load(a )
						logger.info(F"Model loaded from {input_model_file}" )
				elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
						SCREAMING_SNAKE_CASE_				:int				     =						(
						    F"{MODEL_NAME}_rank{accelerator.process_index}.bin"
						    if model_index == 0
						    else F"{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"
						)
						SCREAMING_SNAKE_CASE_				:int				     =						os.path.join(a     , a )
						logger.info(F"Loading model from {input_model_file}" )
						SCREAMING_SNAKE_CASE_				:Dict				     =						torch.load(a )
						logger.info(F"Model loaded from {input_model_file}" )
				elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
						SCREAMING_SNAKE_CASE_				:List[str]				     =						(
						    os.path.join(a     , F"{MODEL_NAME}_{model_index}" )
						    if F"{MODEL_NAME}" not in input_dir
						    else input_dir
						)
						logger.info(F"Loading model from {ckpt_dir}" )
						SCREAMING_SNAKE_CASE_				:Any				     =						{"model": model.state_dict()}
						dist_cp.load_state_dict(
						    state_dict=a     , storage_reader=dist_cp.FileSystemReader(a )     , planner=DefaultLoadPlanner()     , )
						SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						state_dict["model"]
						logger.info(F"Model loaded from {ckpt_dir}" )
				model.load_state_dict(a )


def     lowercase	(    a     , a     , a     , a     , a     , a=0 ):



		'''simple docstring'''




		os.makedirs(a     , exist_ok=a )
		with FSDP.state_dict_type(
		    a     , fsdp_plugin.state_dict_type     , fsdp_plugin.state_dict_config     , fsdp_plugin.optim_state_dict_config ):
				SCREAMING_SNAKE_CASE_				:str				     =						FSDP.optim_state_dict(a     , a )
				if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
						if accelerator.process_index == 0:
								SCREAMING_SNAKE_CASE_				:List[str]				     =						(
								    F"{OPTIMIZER_NAME}.bin" if optimizer_index == 0 else F"{OPTIMIZER_NAME}_{optimizer_index}.bin"
								)
								SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						os.path.join(a     , a )
								logger.info(F"Saving Optimizer state to {output_optimizer_file}" )
								torch.save(a     , a )
								logger.info(F"Optimizer state saved in {output_optimizer_file}" )
				else:
						SCREAMING_SNAKE_CASE_				:Any				     =						os.path.join(a     , F"{OPTIMIZER_NAME}_{optimizer_index}" )
						os.makedirs(a     , exist_ok=a )
						logger.info(F"Saving Optimizer state to {ckpt_dir}" )
						dist_cp.save_state_dict(
						    state_dict={"optimizer": optim_state}     , storage_writer=dist_cp.FileSystemWriter(a )     , planner=DefaultSavePlanner()     , )
						logger.info(F"Optimizer state saved in {ckpt_dir}" )


def     lowercase	(    a     , a     , a     , a     , a     , a=0 ):



		'''simple docstring'''




		accelerator.wait_for_everyone()
		with FSDP.state_dict_type(
		    a     , fsdp_plugin.state_dict_type     , fsdp_plugin.state_dict_config     , fsdp_plugin.optim_state_dict_config ):
				if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
						SCREAMING_SNAKE_CASE_				:List[str]				     =						None
						# below check should work but currently it isn't working (mostly opytorch issue),
						# in the meantime disabling it at the cost of excess memory usage
						# if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
						SCREAMING_SNAKE_CASE_				:List[Any]				     =						(
						    F"{OPTIMIZER_NAME}.bin" if optimizer_index == 0 else F"{OPTIMIZER_NAME}_{optimizer_index}.bin"
						)
						SCREAMING_SNAKE_CASE_				:Dict				     =						os.path.join(a     , a )
						logger.info(F"Loading Optimizer state from {input_optimizer_file}" )
						SCREAMING_SNAKE_CASE_				:str				     =						torch.load(a )
						logger.info(F"Optimizer state loaded from {input_optimizer_file}" )
				else:
						SCREAMING_SNAKE_CASE_				:Any				     =						(
						    os.path.join(a     , F"{OPTIMIZER_NAME}_{optimizer_index}" )
						    if F"{OPTIMIZER_NAME}" not in input_dir
						    else input_dir
						)
						logger.info(F"Loading Optimizer from {ckpt_dir}" )
						SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						load_sharded_optimizer_state_dict(
						    model_state_dict=model.state_dict()     , optimizer_key="optimizer"     , storage_reader=dist_cp.FileSystemReader(a )     , )
						SCREAMING_SNAKE_CASE_				:List[str]				     =						optim_state["optimizer"]
						logger.info(F"Optimizer loaded from {ckpt_dir}" )
				SCREAMING_SNAKE_CASE_				:int				     =						FSDP.optim_state_dict_to_load(a     , a     , a )
				optimizer.load_state_dict(a )






