






'''simple docstring'''
import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version('>=', FSDP_PYTORCH_VERSION):
 import torch.distributed.checkpoint as dist_cp
 from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
 from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
 from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
 from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


UpperCamelCase__			=      get_logger(__name__)




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase=0				):


   """simple docstring"""




   os.makedirs(_UpperCamelCase    ,		exist_ok=_UpperCamelCase				)
   with FSDP.state_dict_type(
       _UpperCamelCase    ,		fsdp_plugin.state_dict_type    ,		fsdp_plugin.state_dict_config    ,		fsdp_plugin.optim_state_dict_config				):
      lowercase_						:     Optional[int]							     =  model.state_dict()
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         lowercase_						:     int							     =  F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
         lowercase_						:     int							     =  os.path.join(_UpperCamelCase    ,		_UpperCamelCase				)
         if accelerator.process_index == 0:
            logger.info(F"""Saving model to {output_model_file}"""				)
            torch.save(_UpperCamelCase    ,		_UpperCamelCase				)
            logger.info(F"""Model saved to {output_model_file}"""				)
      elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
         lowercase_						:     str							     =  (
             F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
             if model_index == 0
             else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
         )
         lowercase_						:     int							     =  os.path.join(_UpperCamelCase    ,		_UpperCamelCase				)
         logger.info(F"""Saving model to {output_model_file}"""				)
         torch.save(_UpperCamelCase    ,		_UpperCamelCase				)
         logger.info(F"""Model saved to {output_model_file}"""				)
      elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
         lowercase_						:     List[str]							     =  os.path.join(_UpperCamelCase    ,		F"""{MODEL_NAME}_{model_index}"""				)
         os.makedirs(_UpperCamelCase    ,		exist_ok=_UpperCamelCase				)
         logger.info(F"""Saving model to {ckpt_dir}"""				)
         lowercase_						:     Union[str, Any]							     =  {"model": state_dict}

         dist_cp.save_state_dict(
             state_dict=_UpperCamelCase    ,		storage_writer=dist_cp.FileSystemWriter(_UpperCamelCase				)    ,		planner=DefaultSavePlanner()    ,		)
         logger.info(F"""Model saved to {ckpt_dir}"""				)




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase=0				):


   """simple docstring"""




   accelerator.wait_for_everyone()
   with FSDP.state_dict_type(
       _UpperCamelCase    ,		fsdp_plugin.state_dict_type    ,		fsdp_plugin.state_dict_config    ,		fsdp_plugin.optim_state_dict_config				):
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         if type(_UpperCamelCase				) != FSDP and accelerator.process_index != 0:
            if not fsdp_plugin.sync_module_states:
               raise ValueError(
                   "Set the `sync_module_states` flag to `True` so that model states are synced across processes when "
                   "initializing FSDP object"				)
            return
         lowercase_						:     List[Any]							     =  F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
         lowercase_						:     int							     =  os.path.join(_UpperCamelCase    ,		_UpperCamelCase				)
         logger.info(F"""Loading model from {input_model_file}"""				)
         lowercase_						:     Tuple							     =  torch.load(_UpperCamelCase				)
         logger.info(F"""Model loaded from {input_model_file}"""				)
      elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
         lowercase_						:     int							     =  (
             F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
             if model_index == 0
             else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
         )
         lowercase_						:     Optional[Any]							     =  os.path.join(_UpperCamelCase    ,		_UpperCamelCase				)
         logger.info(F"""Loading model from {input_model_file}"""				)
         lowercase_						:     str							     =  torch.load(_UpperCamelCase				)
         logger.info(F"""Model loaded from {input_model_file}"""				)
      elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
         lowercase_						:     Any							     =  (
             os.path.join(_UpperCamelCase    ,		F"""{MODEL_NAME}_{model_index}"""				)
             if F"""{MODEL_NAME}""" not in input_dir
             else input_dir
         )
         logger.info(F"""Loading model from {ckpt_dir}"""				)
         lowercase_						:     Optional[int]							     =  {"model": model.state_dict()}
         dist_cp.load_state_dict(
             state_dict=_UpperCamelCase    ,		storage_reader=dist_cp.FileSystemReader(_UpperCamelCase				)    ,		planner=DefaultLoadPlanner()    ,		)
         lowercase_						:     int							     =  state_dict["model"]
         logger.info(F"""Model loaded from {ckpt_dir}"""				)
      model.load_state_dict(_UpperCamelCase				)




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase=0				):


   """simple docstring"""




   os.makedirs(_UpperCamelCase    ,		exist_ok=_UpperCamelCase				)
   with FSDP.state_dict_type(
       _UpperCamelCase    ,		fsdp_plugin.state_dict_type    ,		fsdp_plugin.state_dict_config    ,		fsdp_plugin.optim_state_dict_config				):
      lowercase_						:     Optional[int]							     =  FSDP.optim_state_dict(_UpperCamelCase    ,		_UpperCamelCase				)
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         if accelerator.process_index == 0:
            lowercase_						:     Tuple							     =  (
                F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
            )
            lowercase_						:     Any							     =  os.path.join(_UpperCamelCase    ,		_UpperCamelCase				)
            logger.info(F"""Saving Optimizer state to {output_optimizer_file}"""				)
            torch.save(_UpperCamelCase    ,		_UpperCamelCase				)
            logger.info(F"""Optimizer state saved in {output_optimizer_file}"""				)
      else:
         lowercase_						:     Optional[Any]							     =  os.path.join(_UpperCamelCase    ,		F"""{OPTIMIZER_NAME}_{optimizer_index}"""				)
         os.makedirs(_UpperCamelCase    ,		exist_ok=_UpperCamelCase				)
         logger.info(F"""Saving Optimizer state to {ckpt_dir}"""				)
         dist_cp.save_state_dict(
             state_dict={"optimizer": optim_state}    ,		storage_writer=dist_cp.FileSystemWriter(_UpperCamelCase				)    ,		planner=DefaultSavePlanner()    ,		)
         logger.info(F"""Optimizer state saved in {ckpt_dir}"""				)





def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase=0				):


   """simple docstring"""




   accelerator.wait_for_everyone()
   with FSDP.state_dict_type(
       _UpperCamelCase    ,		fsdp_plugin.state_dict_type    ,		fsdp_plugin.state_dict_config    ,		fsdp_plugin.optim_state_dict_config				):
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         lowercase_						:     Dict							     =  None
         # below check should work but currently it isn't working (mostly opytorch issue),
         # in the meantime disabling it at the cost of excess memory usage
         # if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
         lowercase_						:     Optional[int]							     =  (
             F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
         )
         lowercase_						:     str							     =  os.path.join(_UpperCamelCase    ,		_UpperCamelCase				)
         logger.info(F"""Loading Optimizer state from {input_optimizer_file}"""				)
         lowercase_						:     int							     =  torch.load(_UpperCamelCase				)
         logger.info(F"""Optimizer state loaded from {input_optimizer_file}"""				)
      else:
         lowercase_						:     Any							     =  (
             os.path.join(_UpperCamelCase    ,		F"""{OPTIMIZER_NAME}_{optimizer_index}"""				)
             if F"""{OPTIMIZER_NAME}""" not in input_dir
             else input_dir
         )
         logger.info(F"""Loading Optimizer from {ckpt_dir}"""				)
         lowercase_						:     Optional[int]							     =  load_sharded_optimizer_state_dict(
             model_state_dict=model.state_dict()    ,		optimizer_key="optimizer"    ,		storage_reader=dist_cp.FileSystemReader(_UpperCamelCase				)    ,		)
         lowercase_						:     Optional[Any]							     =  optim_state["optimizer"]
         logger.info(F"""Optimizer loaded from {ckpt_dir}"""				)
      lowercase_						:     Union[str, Any]							     =  FSDP.optim_state_dict_to_load(_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase				)
      optimizer.load_state_dict(_UpperCamelCase				)












'''simple docstring'''
import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
 from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
 from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
 from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


UpperCamelCase__			=      DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

UpperCamelCase__			=      'main'
# Default branch name
UpperCamelCase__			=      'f2c752cfc5c0ab6f4bdec59acea69eefbee381c2'
# One particular commit (not the top of `main`)
UpperCamelCase__			=      'aaaaaaa'
# This commit does not exist, so we should 404.

UpperCamelCase__			=      'd9e9f15bc825e4b2c9249e9578f884bbcb5e3684'
# Sha-1 of config.json on the top of `main`, for checking purposes
UpperCamelCase__			=      '4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3'




@contextlib.contextmanager
def 		__SCREAMING_SNAKE_CASE       (	):


   """simple docstring"""




   print("Welcome!"				)
   yield
   print("Bye!"				)




@contextlib.contextmanager
def 		__SCREAMING_SNAKE_CASE       (	):


   """simple docstring"""




   print("Bonjour!"				)
   yield
   print("Au revoir!"				)


class 							_UpperCAmelCase    (		unittest.TestCase      ):




    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       assert transformers.__spec__ is not None
       assert importlib.util.find_spec("transformers"      ) is not None


class 							_UpperCAmelCase    (		unittest.TestCase      ):




    @unittest.mock.patch("sys.stdout"	,  new_callable=io.StringIO      )
    def 					lowerCAmelCase__					(      self	:							str	,  a	:							List[str]      ):






       '''simple docstring'''






       with ContextManagers([]      ):
          print("Transformers are awesome!"      )
       # The print statement adds a new line at the end of the output
       self.assertEqual(mock_stdout.getvalue()	,  "Transformers are awesome!\n"      )




    @unittest.mock.patch("sys.stdout"	,  new_callable=io.StringIO      )
    def 					lowerCAmelCase__					(      self	:							List[str]	,  a	:							Optional[int]      ):






       '''simple docstring'''






       with ContextManagers([context_en()]      ):
          print("Transformers are awesome!"      )
       # The output should be wrapped with an English welcome and goodbye
       self.assertEqual(mock_stdout.getvalue()	,  "Welcome!\nTransformers are awesome!\nBye!\n"      )




    @unittest.mock.patch("sys.stdout"	,  new_callable=io.StringIO      )
    def 					lowerCAmelCase__					(      self	:							Tuple	,  a	:							List[Any]      ):






       '''simple docstring'''






       with ContextManagers([context_fr(), context_en()]      ):
          print("Transformers are awesome!"      )
       # The output should be wrapped with an English and French welcome and goodbye
       self.assertEqual(mock_stdout.getvalue()	,  "Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n"      )




    @require_torch
    def 					lowerCAmelCase__					(      self	:							Tuple      ):






       '''simple docstring'''






       self.assertEqual(find_labels(a      )	,  ["labels"]      )
       self.assertEqual(find_labels(a      )	,  ["labels", "next_sentence_label"]      )
       self.assertEqual(find_labels(a      )	,  ["start_positions", "end_positions"]      )


       class 							_UpperCAmelCase    (		snake_case      ):
           pass


       self.assertEqual(find_labels(a      )	,  ["labels"]      )




    @require_tf
    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       self.assertEqual(find_labels(a      )	,  ["labels"]      )
       self.assertEqual(find_labels(a      )	,  ["labels", "next_sentence_label"]      )
       self.assertEqual(find_labels(a      )	,  ["start_positions", "end_positions"]      )


       class 							_UpperCAmelCase    (		snake_case      ):
           pass


       self.assertEqual(find_labels(a      )	,  ["labels"]      )





    @require_flax
    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       self.assertEqual(find_labels(a      )	,  []      )
       self.assertEqual(find_labels(a      )	,  []      )
       self.assertEqual(find_labels(a      )	,  []      )


       class 							_UpperCAmelCase    (		snake_case      ):
           pass


       self.assertEqual(find_labels(a      )	,  []      )




