




import unittest

from transformers import TrOCRConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers.models.trocr.modeling_trocr import TrOCRDecoder, TrOCRForCausalLM
@require_torch
class 			a__    :






						def __init__(   self		,			_A		,			_A=9_9		,			_A=1_3		,			_A=1_6		,			_A=7		,			_A=True		,			_A=True		,			_A=True		,			_A=False		,			_A=True		,			_A=2		,			_A=3_2		,			_A=4		,			_A=4		,			_A=3_0		,			_A=0		,			_A=1		,			_A=2		,			_A=None		,			):




											"""simple docstring"""



											__lowerCAmelCase				   =       parent
											__lowerCAmelCase				   =       batch_size
											__lowerCAmelCase				   =       decoder_seq_length
											# For common tests
											__lowerCAmelCase				   =       self.decoder_seq_length
											__lowerCAmelCase				   =       is_training
											__lowerCAmelCase				   =       use_attention_mask
											__lowerCAmelCase				   =       use_labels

											__lowerCAmelCase				   =       vocab_size
											__lowerCAmelCase				   =       d_model
											__lowerCAmelCase				   =       d_model
											__lowerCAmelCase				   =       decoder_layers
											__lowerCAmelCase				   =       decoder_layers
											__lowerCAmelCase				   =       decoder_ffn_dim
											__lowerCAmelCase				   =       decoder_attention_heads
											__lowerCAmelCase				   =       decoder_attention_heads
											__lowerCAmelCase				   =       eos_token_id
											__lowerCAmelCase				   =       bos_token_id
											__lowerCAmelCase				   =       pad_token_id
											__lowerCAmelCase				   =       decoder_start_token_id
											__lowerCAmelCase				   =       use_cache
											__lowerCAmelCase				   =       max_position_embeddings

											__lowerCAmelCase				   =       None
											__lowerCAmelCase				   =       decoder_seq_length
											__lowerCAmelCase				   =       2
											__lowerCAmelCase				   =       1






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       ids_tensor([self.batch_size, self.decoder_seq_length]		,			self.vocab_size			)

											__lowerCAmelCase				   =       None
											if self.use_attention_mask:
																__lowerCAmelCase				   =       ids_tensor([self.batch_size, self.decoder_seq_length]		,			vocab_size=2			)

											__lowerCAmelCase				   =       None
											if self.use_labels:
																__lowerCAmelCase				   =       ids_tensor([self.batch_size, self.decoder_seq_length]		,			self.vocab_size			)

											__lowerCAmelCase				   =       TrOCRConfig(
											    vocab_size=self.vocab_size		,			d_model=self.d_model		,			decoder_layers=self.decoder_layers		,			decoder_ffn_dim=self.decoder_ffn_dim		,			decoder_attention_heads=self.decoder_attention_heads		,			eos_token_id=self.eos_token_id		,			bos_token_id=self.bos_token_id		,			use_cache=self.use_cache		,			pad_token_id=self.pad_token_id		,			decoder_start_token_id=self.decoder_start_token_id		,			max_position_embeddings=self.max_position_embeddings		,			)

											return (config, input_ids, attention_mask, lm_labels)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A		,			_A		,			_A		,			):




											"""simple docstring"""



											__lowerCAmelCase				   =       True
											__lowerCAmelCase				   =       TrOCRDecoder(config=_A			).to(_A			).eval()
											__lowerCAmelCase				   =       input_ids[:2]

											input_ids[input_ids == 0] += 1
											# first forward pass
											__lowerCAmelCase				   =       model(_A		,			use_cache=_A			)
											__lowerCAmelCase				   =       model(_A			)
											__lowerCAmelCase				   =       model(_A		,			use_cache=_A			)

											self.parent.assertTrue(len(_A			) == len(_A			)			)
											self.parent.assertTrue(len(_A			) == len(_A			) + 1			)

											__lowerCAmelCase				   =       outputs["past_key_values"]

											# create hypothetical next token and extent to next_input_ids
											__lowerCAmelCase				   =       ids_tensor((2, 1)		,			config.vocab_size - 1			) + 1

											# append to next input_ids and
											__lowerCAmelCase				   =       torch.cat([input_ids, next_tokens]		,			dim=-1			)

											__lowerCAmelCase				   =       model(_A			)["last_hidden_state"]
											__lowerCAmelCase				   =       model(_A		,			past_key_values=_A			)["last_hidden_state"]

											# select random slice
											__lowerCAmelCase				   =       ids_tensor((1,)		,			output_from_past.shape[-1]			).item()
											__lowerCAmelCase				   =       output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
											__lowerCAmelCase				   =       output_from_past[:, 0, random_slice_idx].detach()

											# test that outputs are equal for slice
											assert torch.allclose(_A		,			_A		,			atol=1E-3			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       self.prepare_config_and_inputs()
											__lowerCAmelCase   ,							__lowerCAmelCase   ,							__lowerCAmelCase   ,							__lowerCAmelCase				   =       config_and_inputs

											__lowerCAmelCase				   =       {"input_ids": input_ids, "attention_mask": attention_mask}
											return config, inputs_dict






@require_torch
class 			a__    (					snake_case__ ,				snake_case__ ,				snake_case__ ,				unittest.TestCase      ):
						_a							:    List[Any]					=    (TrOCRDecoder, TrOCRForCausalLM) if is_torch_available() else ()
						_a							:    Dict					=    (TrOCRForCausalLM,) if is_torch_available() else ()
						_a							:    Union[str, Any]					=    {"""text-generation""": TrOCRForCausalLM} if is_torch_available() else {}
						_a							:    List[Any]					=    True
						_a							:    Dict					=    False






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       TrOCRStandaloneDecoderModelTester(self		,			is_training=_A			)
											__lowerCAmelCase				   =       ConfigTester(self		,			config_class=_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											self.config_tester.run_common_tests()






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_decoder_model_past(*_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											return






						@unittest.skip("The model doesn't support left padding"			)  # and it's not used enough to be worth fixing :)
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											pass










import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version(""">=""", FSDP_PYTORCH_VERSION):
					import torch.distributed.checkpoint as dist_cp
					from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
					from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
					from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
					from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


UpperCamelCase__    			=			get_logger(__name__)







def 	_a      (						SCREAMING_SNAKE_CASE_							:				List[str]      ,      SCREAMING_SNAKE_CASE_							:				Tuple      ,      SCREAMING_SNAKE_CASE_							:				Tuple      ,      SCREAMING_SNAKE_CASE_							:				Optional[Any]      ,      SCREAMING_SNAKE_CASE_							:				str=0     ):
					os.makedirs(SCREAMING_SNAKE_CASE_      ,      exist_ok=SCREAMING_SNAKE_CASE_     )
					with FSDP.state_dict_type(
					    SCREAMING_SNAKE_CASE_      ,      fsdp_plugin.state_dict_type      ,      fsdp_plugin.state_dict_config      ,      fsdp_plugin.optim_state_dict_config     ):
										__lowerCAmelCase				   =       model.state_dict()
										if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
															__lowerCAmelCase				   =       F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
															__lowerCAmelCase				   =       os.path.join(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
															if accelerator.process_index == 0:
																				logger.info(F"""Saving model to {output_model_file}"""     )
																				torch.save(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
																				logger.info(F"""Model saved to {output_model_file}"""     )
										elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
															__lowerCAmelCase				   =       (
															    F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
															    if model_index == 0
															    else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
															)
															__lowerCAmelCase				   =       os.path.join(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
															logger.info(F"""Saving model to {output_model_file}"""     )
															torch.save(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
															logger.info(F"""Model saved to {output_model_file}"""     )
										elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
															__lowerCAmelCase				   =       os.path.join(SCREAMING_SNAKE_CASE_      ,      F"""{MODEL_NAME}_{model_index}"""     )
															os.makedirs(SCREAMING_SNAKE_CASE_      ,      exist_ok=SCREAMING_SNAKE_CASE_     )
															logger.info(F"""Saving model to {ckpt_dir}"""     )
															__lowerCAmelCase				   =       {"model": state_dict}

															dist_cp.save_state_dict(
															    state_dict=SCREAMING_SNAKE_CASE_      ,      storage_writer=dist_cp.FileSystemWriter(SCREAMING_SNAKE_CASE_     )      ,      planner=DefaultSavePlanner()      ,      )
															logger.info(F"""Model saved to {ckpt_dir}"""     )







def 	_a      (						SCREAMING_SNAKE_CASE_							:				Dict      ,      SCREAMING_SNAKE_CASE_							:				int      ,      SCREAMING_SNAKE_CASE_							:				Optional[int]      ,      SCREAMING_SNAKE_CASE_							:				List[str]      ,      SCREAMING_SNAKE_CASE_							:				Any=0     ):
					accelerator.wait_for_everyone()
					with FSDP.state_dict_type(
					    SCREAMING_SNAKE_CASE_      ,      fsdp_plugin.state_dict_type      ,      fsdp_plugin.state_dict_config      ,      fsdp_plugin.optim_state_dict_config     ):
										if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
															if type(SCREAMING_SNAKE_CASE_     ) != FSDP and accelerator.process_index != 0:
																				if not fsdp_plugin.sync_module_states:
																									raise ValueError(
																									    "Set the `sync_module_states` flag to `True` so that model states are synced across processes when "
																									    "initializing FSDP object"     )
																				return
															__lowerCAmelCase				   =       F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
															__lowerCAmelCase				   =       os.path.join(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
															logger.info(F"""Loading model from {input_model_file}"""     )
															__lowerCAmelCase				   =       torch.load(SCREAMING_SNAKE_CASE_     )
															logger.info(F"""Model loaded from {input_model_file}"""     )
										elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
															__lowerCAmelCase				   =       (
															    F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
															    if model_index == 0
															    else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
															)
															__lowerCAmelCase				   =       os.path.join(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
															logger.info(F"""Loading model from {input_model_file}"""     )
															__lowerCAmelCase				   =       torch.load(SCREAMING_SNAKE_CASE_     )
															logger.info(F"""Model loaded from {input_model_file}"""     )
										elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
															__lowerCAmelCase				   =       (
															    os.path.join(SCREAMING_SNAKE_CASE_      ,      F"""{MODEL_NAME}_{model_index}"""     )
															    if F"""{MODEL_NAME}""" not in input_dir
															    else input_dir
															)
															logger.info(F"""Loading model from {ckpt_dir}"""     )
															__lowerCAmelCase				   =       {"model": model.state_dict()}
															dist_cp.load_state_dict(
															    state_dict=SCREAMING_SNAKE_CASE_      ,      storage_reader=dist_cp.FileSystemReader(SCREAMING_SNAKE_CASE_     )      ,      planner=DefaultLoadPlanner()      ,      )
															__lowerCAmelCase				   =       state_dict["model"]
															logger.info(F"""Model loaded from {ckpt_dir}"""     )
										model.load_state_dict(SCREAMING_SNAKE_CASE_     )







def 	_a      (						SCREAMING_SNAKE_CASE_							:				Dict      ,      SCREAMING_SNAKE_CASE_							:				int      ,      SCREAMING_SNAKE_CASE_							:				List[str]      ,      SCREAMING_SNAKE_CASE_							:				Optional[Any]      ,      SCREAMING_SNAKE_CASE_							:				Optional[int]      ,      SCREAMING_SNAKE_CASE_							:				str=0     ):
					os.makedirs(SCREAMING_SNAKE_CASE_      ,      exist_ok=SCREAMING_SNAKE_CASE_     )
					with FSDP.state_dict_type(
					    SCREAMING_SNAKE_CASE_      ,      fsdp_plugin.state_dict_type      ,      fsdp_plugin.state_dict_config      ,      fsdp_plugin.optim_state_dict_config     ):
										__lowerCAmelCase				   =       FSDP.optim_state_dict(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
										if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
															if accelerator.process_index == 0:
																				__lowerCAmelCase				   =       (
																				    F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
																				)
																				__lowerCAmelCase				   =       os.path.join(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
																				logger.info(F"""Saving Optimizer state to {output_optimizer_file}"""     )
																				torch.save(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
																				logger.info(F"""Optimizer state saved in {output_optimizer_file}"""     )
										else:
															__lowerCAmelCase				   =       os.path.join(SCREAMING_SNAKE_CASE_      ,      F"""{OPTIMIZER_NAME}_{optimizer_index}"""     )
															os.makedirs(SCREAMING_SNAKE_CASE_      ,      exist_ok=SCREAMING_SNAKE_CASE_     )
															logger.info(F"""Saving Optimizer state to {ckpt_dir}"""     )
															dist_cp.save_state_dict(
															    state_dict={"optimizer": optim_state}      ,      storage_writer=dist_cp.FileSystemWriter(SCREAMING_SNAKE_CASE_     )      ,      planner=DefaultSavePlanner()      ,      )
															logger.info(F"""Optimizer state saved in {ckpt_dir}"""     )







def 	_a      (						SCREAMING_SNAKE_CASE_							:				Dict      ,      SCREAMING_SNAKE_CASE_							:				Dict      ,      SCREAMING_SNAKE_CASE_							:				Tuple      ,      SCREAMING_SNAKE_CASE_							:				List[str]      ,      SCREAMING_SNAKE_CASE_							:				str      ,      SCREAMING_SNAKE_CASE_							:				Dict=0     ):
					accelerator.wait_for_everyone()
					with FSDP.state_dict_type(
					    SCREAMING_SNAKE_CASE_      ,      fsdp_plugin.state_dict_type      ,      fsdp_plugin.state_dict_config      ,      fsdp_plugin.optim_state_dict_config     ):
										if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
															__lowerCAmelCase				   =       None
															# below check should work but currently it isn't working (mostly opytorch issue),
															# in the meantime disabling it at the cost of excess memory usage
															# if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
															__lowerCAmelCase				   =       (
															    F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
															)
															__lowerCAmelCase				   =       os.path.join(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
															logger.info(F"""Loading Optimizer state from {input_optimizer_file}"""     )
															__lowerCAmelCase				   =       torch.load(SCREAMING_SNAKE_CASE_     )
															logger.info(F"""Optimizer state loaded from {input_optimizer_file}"""     )
										else:
															__lowerCAmelCase				   =       (
															    os.path.join(SCREAMING_SNAKE_CASE_      ,      F"""{OPTIMIZER_NAME}_{optimizer_index}"""     )
															    if F"""{OPTIMIZER_NAME}""" not in input_dir
															    else input_dir
															)
															logger.info(F"""Loading Optimizer from {ckpt_dir}"""     )
															__lowerCAmelCase				   =       load_sharded_optimizer_state_dict(
															    model_state_dict=model.state_dict()      ,      optimizer_key="optimizer"      ,      storage_reader=dist_cp.FileSystemReader(SCREAMING_SNAKE_CASE_     )      ,      )
															__lowerCAmelCase				   =       optim_state["optimizer"]
															logger.info(F"""Optimizer loaded from {ckpt_dir}"""     )
										__lowerCAmelCase				   =       FSDP.optim_state_dict_to_load(SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
										optimizer.load_state_dict(SCREAMING_SNAKE_CASE_     )




