

'''simple docstring'''





UpperCamelCase_    						=       {
    """Pillow""": """Pillow<10.0.0""",
    """accelerate""": """accelerate>=0.20.3""",
    """av""": """av==9.2.0""",
    """beautifulsoup4""": """beautifulsoup4""",
    """black""": """black~=23.1""",
    """codecarbon""": """codecarbon==1.2.0""",
    """cookiecutter""": """cookiecutter==1.7.3""",
    """dataclasses""": """dataclasses""",
    """datasets""": """datasets!=2.5.0""",
    """decord""": """decord==0.6.0""",
    """deepspeed""": """deepspeed>=0.9.3""",
    """diffusers""": """diffusers""",
    """dill""": """dill<0.3.5""",
    """evaluate""": """evaluate>=0.2.0""",
    """fairscale""": """fairscale>0.3""",
    """faiss-cpu""": """faiss-cpu""",
    """fastapi""": """fastapi""",
    """filelock""": """filelock""",
    """flax""": """flax>=0.4.1,<=0.7.0""",
    """ftfy""": """ftfy""",
    """fugashi""": """fugashi>=1.0""",
    """GitPython""": """GitPython<3.1.19""",
    """hf-doc-builder""": """hf-doc-builder>=0.3.0""",
    """huggingface-hub""": """huggingface-hub>=0.14.1,<1.0""",
    """importlib_metadata""": """importlib_metadata""",
    """ipadic""": """ipadic>=1.0.0,<2.0""",
    """isort""": """isort>=5.5.4""",
    """jax""": """jax>=0.2.8,!=0.3.2,<=0.4.13""",
    """jaxlib""": """jaxlib>=0.1.65,<=0.4.13""",
    """jieba""": """jieba""",
    """kenlm""": """kenlm""",
    """keras-nlp""": """keras-nlp>=0.3.1""",
    """librosa""": """librosa""",
    """nltk""": """nltk""",
    """natten""": """natten>=0.14.6""",
    """numpy""": """numpy>=1.17""",
    """onnxconverter-common""": """onnxconverter-common""",
    """onnxruntime-tools""": """onnxruntime-tools>=1.4.2""",
    """onnxruntime""": """onnxruntime>=1.4.0""",
    """opencv-python""": """opencv-python""",
    """optuna""": """optuna""",
    """optax""": """optax>=0.0.8,<=0.1.4""",
    """packaging""": """packaging>=20.0""",
    """parameterized""": """parameterized""",
    """phonemizer""": """phonemizer""",
    """protobuf""": """protobuf""",
    """psutil""": """psutil""",
    """pyyaml""": """pyyaml>=5.1""",
    """pydantic""": """pydantic<2""",
    """pytest""": """pytest>=7.2.0""",
    """pytest-timeout""": """pytest-timeout""",
    """pytest-xdist""": """pytest-xdist""",
    """python""": """python>=3.8.0""",
    """ray[tune]""": """ray[tune]""",
    """regex""": """regex!=2019.12.17""",
    """requests""": """requests""",
    """rhoknp""": """rhoknp>=1.1.0,<1.3.1""",
    """rjieba""": """rjieba""",
    """rouge-score""": """rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1""",
    """ruff""": """ruff>=0.0.241,<=0.0.259""",
    """sacrebleu""": """sacrebleu>=1.4.12,<2.0.0""",
    """sacremoses""": """sacremoses""",
    """safetensors""": """safetensors>=0.3.1""",
    """sagemaker""": """sagemaker>=2.31.0""",
    """scikit-learn""": """scikit-learn""",
    """sentencepiece""": """sentencepiece>=0.1.91,!=0.1.92""",
    """sigopt""": """sigopt""",
    """starlette""": """starlette""",
    """sudachipy""": """sudachipy>=0.6.6""",
    """sudachidict_core""": """sudachidict_core>=20220729""",
    """tensorflow-cpu""": """tensorflow-cpu>=2.6,<2.14""",
    """tensorflow""": """tensorflow>=2.6,<2.14""",
    """tensorflow-text""": """tensorflow-text<2.14""",
    """tf2onnx""": """tf2onnx""",
    """timeout-decorator""": """timeout-decorator""",
    """timm""": """timm""",
    """tokenizers""": """tokenizers>=0.11.1,!=0.11.3,<0.14""",
    """torch""": """torch>=1.9,!=1.12.0""",
    """torchaudio""": """torchaudio""",
    """torchvision""": """torchvision""",
    """pyctcdecode""": """pyctcdecode>=0.4.0""",
    """tqdm""": """tqdm>=4.27""",
    """unidic""": """unidic>=1.0.2""",
    """unidic_lite""": """unidic_lite>=1.0.7""",
    """urllib3""": """urllib3<2.0.0""",
    """uvicorn""": """uvicorn""",
}












import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version('>=', FSDP_PYTORCH_VERSION):
    import torch.distributed.checkpoint as dist_cp
    from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
    from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
    from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
    from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


UpperCamelCase_												=	get_logger(__name__)






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A				,							A=0     ):




   '''simple docstring'''

   os.makedirs(A				,							exist_ok=A     )
   with FSDP.state_dict_type(
       A				,							fsdp_plugin.state_dict_type				,							fsdp_plugin.state_dict_config				,							fsdp_plugin.optim_state_dict_config     ):
      UpperCAmelCase__						=model.state_dict()
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         UpperCAmelCase__						=F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
         UpperCAmelCase__						=os.path.join(A				,							A     )
         if accelerator.process_index == 0:
            logger.info(F"""Saving model to {output_model_file}"""     )
            torch.save(A				,							A     )
            logger.info(F"""Model saved to {output_model_file}"""     )
      elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
         UpperCAmelCase__						=(
             F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
             if model_index == 0
             else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
         )
         UpperCAmelCase__						=os.path.join(A				,							A     )
         logger.info(F"""Saving model to {output_model_file}"""     )
         torch.save(A				,							A     )
         logger.info(F"""Model saved to {output_model_file}"""     )
      elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
         UpperCAmelCase__						=os.path.join(A				,							F"""{MODEL_NAME}_{model_index}"""     )
         os.makedirs(A				,							exist_ok=A     )
         logger.info(F"""Saving model to {ckpt_dir}"""     )
         UpperCAmelCase__						={"model": state_dict}

         dist_cp.save_state_dict(
             state_dict=A				,							storage_writer=dist_cp.FileSystemWriter(A     )				,							planner=DefaultSavePlanner()				,							)
         logger.info(F"""Model saved to {ckpt_dir}"""     )






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A				,							A=0     ):




   '''simple docstring'''

   accelerator.wait_for_everyone()
   with FSDP.state_dict_type(
       A				,							fsdp_plugin.state_dict_type				,							fsdp_plugin.state_dict_config				,							fsdp_plugin.optim_state_dict_config     ):
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         if type(A     ) != FSDP and accelerator.process_index != 0:
            if not fsdp_plugin.sync_module_states:
               raise ValueError(
                   "Set the `sync_module_states` flag to `True` so that model states are synced across processes when "
                   "initializing FSDP object"     )
            return
         UpperCAmelCase__						=F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
         UpperCAmelCase__						=os.path.join(A				,							A     )
         logger.info(F"""Loading model from {input_model_file}"""     )
         UpperCAmelCase__						=torch.load(A     )
         logger.info(F"""Model loaded from {input_model_file}"""     )
      elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
         UpperCAmelCase__						=(
             F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
             if model_index == 0
             else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
         )
         UpperCAmelCase__						=os.path.join(A				,							A     )
         logger.info(F"""Loading model from {input_model_file}"""     )
         UpperCAmelCase__						=torch.load(A     )
         logger.info(F"""Model loaded from {input_model_file}"""     )
      elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
         UpperCAmelCase__						=(
             os.path.join(A				,							F"""{MODEL_NAME}_{model_index}"""     )
             if F"""{MODEL_NAME}""" not in input_dir
             else input_dir
         )
         logger.info(F"""Loading model from {ckpt_dir}"""     )
         UpperCAmelCase__						={"model": model.state_dict()}
         dist_cp.load_state_dict(
             state_dict=A				,							storage_reader=dist_cp.FileSystemReader(A     )				,							planner=DefaultLoadPlanner()				,							)
         UpperCAmelCase__						=state_dict["model"]
         logger.info(F"""Model loaded from {ckpt_dir}"""     )
      model.load_state_dict(A     )






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A				,							A				,							A=0     ):




   '''simple docstring'''

   os.makedirs(A				,							exist_ok=A     )
   with FSDP.state_dict_type(
       A				,							fsdp_plugin.state_dict_type				,							fsdp_plugin.state_dict_config				,							fsdp_plugin.optim_state_dict_config     ):
      UpperCAmelCase__						=FSDP.optim_state_dict(A				,							A     )
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         if accelerator.process_index == 0:
            UpperCAmelCase__						=(
                F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
            )
            UpperCAmelCase__						=os.path.join(A				,							A     )
            logger.info(F"""Saving Optimizer state to {output_optimizer_file}"""     )
            torch.save(A				,							A     )
            logger.info(F"""Optimizer state saved in {output_optimizer_file}"""     )
      else:
         UpperCAmelCase__						=os.path.join(A				,							F"""{OPTIMIZER_NAME}_{optimizer_index}"""     )
         os.makedirs(A				,							exist_ok=A     )
         logger.info(F"""Saving Optimizer state to {ckpt_dir}"""     )
         dist_cp.save_state_dict(
             state_dict={"optimizer": optim_state}				,							storage_writer=dist_cp.FileSystemWriter(A     )				,							planner=DefaultSavePlanner()				,							)
         logger.info(F"""Optimizer state saved in {ckpt_dir}"""     )






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A				,							A				,							A=0     ):




   '''simple docstring'''

   accelerator.wait_for_everyone()
   with FSDP.state_dict_type(
       A				,							fsdp_plugin.state_dict_type				,							fsdp_plugin.state_dict_config				,							fsdp_plugin.optim_state_dict_config     ):
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         UpperCAmelCase__						=None
         # below check should work but currently it isn't working (mostly opytorch issue),
         # in the meantime disabling it at the cost of excess memory usage
         # if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
         UpperCAmelCase__						=(
             F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
         )
         UpperCAmelCase__						=os.path.join(A				,							A     )
         logger.info(F"""Loading Optimizer state from {input_optimizer_file}"""     )
         UpperCAmelCase__						=torch.load(A     )
         logger.info(F"""Optimizer state loaded from {input_optimizer_file}"""     )
      else:
         UpperCAmelCase__						=(
             os.path.join(A				,							F"""{OPTIMIZER_NAME}_{optimizer_index}"""     )
             if F"""{OPTIMIZER_NAME}""" not in input_dir
             else input_dir
         )
         logger.info(F"""Loading Optimizer from {ckpt_dir}"""     )
         UpperCAmelCase__						=load_sharded_optimizer_state_dict(
             model_state_dict=model.state_dict()				,							optimizer_key="optimizer"				,							storage_reader=dist_cp.FileSystemReader(A     )				,							)
         UpperCAmelCase__						=optim_state["optimizer"]
         logger.info(F"""Optimizer loaded from {ckpt_dir}"""     )
      UpperCAmelCase__						=FSDP.optim_state_dict_to_load(A				,							A				,							A     )
      optimizer.load_state_dict(A     )



