





UpperCamelCase_												=	{
    'Pillow': 'Pillow<10.0.0',
    'accelerate': 'accelerate>=0.20.3',
    'av': 'av==9.2.0',
    'beautifulsoup4': 'beautifulsoup4',
    'black': 'black~=23.1',
    'codecarbon': 'codecarbon==1.2.0',
    'cookiecutter': 'cookiecutter==1.7.3',
    'dataclasses': 'dataclasses',
    'datasets': 'datasets!=2.5.0',
    'decord': 'decord==0.6.0',
    'deepspeed': 'deepspeed>=0.9.3',
    'diffusers': 'diffusers',
    'dill': 'dill<0.3.5',
    'evaluate': 'evaluate>=0.2.0',
    'fairscale': 'fairscale>0.3',
    'faiss-cpu': 'faiss-cpu',
    'fastapi': 'fastapi',
    'filelock': 'filelock',
    'flax': 'flax>=0.4.1,<=0.7.0',
    'ftfy': 'ftfy',
    'fugashi': 'fugashi>=1.0',
    'GitPython': 'GitPython<3.1.19',
    'hf-doc-builder': 'hf-doc-builder>=0.3.0',
    'huggingface-hub': 'huggingface-hub>=0.14.1,<1.0',
    'importlib_metadata': 'importlib_metadata',
    'ipadic': 'ipadic>=1.0.0,<2.0',
    'isort': 'isort>=5.5.4',
    'jax': 'jax>=0.2.8,!=0.3.2,<=0.4.13',
    'jaxlib': 'jaxlib>=0.1.65,<=0.4.13',
    'jieba': 'jieba',
    'kenlm': 'kenlm',
    'keras-nlp': 'keras-nlp>=0.3.1',
    'librosa': 'librosa',
    'nltk': 'nltk',
    'natten': 'natten>=0.14.6',
    'numpy': 'numpy>=1.17',
    'onnxconverter-common': 'onnxconverter-common',
    'onnxruntime-tools': 'onnxruntime-tools>=1.4.2',
    'onnxruntime': 'onnxruntime>=1.4.0',
    'opencv-python': 'opencv-python',
    'optuna': 'optuna',
    'optax': 'optax>=0.0.8,<=0.1.4',
    'packaging': 'packaging>=20.0',
    'parameterized': 'parameterized',
    'phonemizer': 'phonemizer',
    'protobuf': 'protobuf',
    'psutil': 'psutil',
    'pyyaml': 'pyyaml>=5.1',
    'pydantic': 'pydantic<2',
    'pytest': 'pytest>=7.2.0',
    'pytest-timeout': 'pytest-timeout',
    'pytest-xdist': 'pytest-xdist',
    'python': 'python>=3.8.0',
    'ray[tune]': 'ray[tune]',
    'regex': 'regex!=2019.12.17',
    'requests': 'requests',
    'rhoknp': 'rhoknp>=1.1.0,<1.3.1',
    'rjieba': 'rjieba',
    'rouge-score': 'rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1',
    'ruff': 'ruff>=0.0.241,<=0.0.259',
    'sacrebleu': 'sacrebleu>=1.4.12,<2.0.0',
    'sacremoses': 'sacremoses',
    'safetensors': 'safetensors>=0.3.1',
    'sagemaker': 'sagemaker>=2.31.0',
    'scikit-learn': 'scikit-learn',
    'sentencepiece': 'sentencepiece>=0.1.91,!=0.1.92',
    'sigopt': 'sigopt',
    'starlette': 'starlette',
    'sudachipy': 'sudachipy>=0.6.6',
    'sudachidict_core': 'sudachidict_core>=20220729',
    'tensorflow-cpu': 'tensorflow-cpu>=2.6,<2.14',
    'tensorflow': 'tensorflow>=2.6,<2.14',
    'tensorflow-text': 'tensorflow-text<2.14',
    'tf2onnx': 'tf2onnx',
    'timeout-decorator': 'timeout-decorator',
    'timm': 'timm',
    'tokenizers': 'tokenizers>=0.11.1,!=0.11.3,<0.14',
    'torch': 'torch>=1.9,!=1.12.0',
    'torchaudio': 'torchaudio',
    'torchvision': 'torchvision',
    'pyctcdecode': 'pyctcdecode>=0.4.0',
    'tqdm': 'tqdm>=4.27',
    'unidic': 'unidic>=1.0.2',
    'unidic_lite': 'unidic_lite>=1.0.7',
    'urllib3': 'urllib3<2.0.0',
    'uvicorn': 'uvicorn',
}










import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version('>=', FSDP_PYTORCH_VERSION):
    import torch.distributed.checkpoint as dist_cp
    from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
    from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
    from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
    from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


UpperCamelCase_												=	get_logger(__name__)






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A				,							A=0     ):




   '''simple docstring'''

   os.makedirs(A				,							exist_ok=A     )
   with FSDP.state_dict_type(
       A				,							fsdp_plugin.state_dict_type				,							fsdp_plugin.state_dict_config				,							fsdp_plugin.optim_state_dict_config     ):
      UpperCAmelCase__						=model.state_dict()
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         UpperCAmelCase__						=F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
         UpperCAmelCase__						=os.path.join(A				,							A     )
         if accelerator.process_index == 0:
            logger.info(F"""Saving model to {output_model_file}"""     )
            torch.save(A				,							A     )
            logger.info(F"""Model saved to {output_model_file}"""     )
      elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
         UpperCAmelCase__						=(
             F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
             if model_index == 0
             else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
         )
         UpperCAmelCase__						=os.path.join(A				,							A     )
         logger.info(F"""Saving model to {output_model_file}"""     )
         torch.save(A				,							A     )
         logger.info(F"""Model saved to {output_model_file}"""     )
      elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
         UpperCAmelCase__						=os.path.join(A				,							F"""{MODEL_NAME}_{model_index}"""     )
         os.makedirs(A				,							exist_ok=A     )
         logger.info(F"""Saving model to {ckpt_dir}"""     )
         UpperCAmelCase__						={"model": state_dict}

         dist_cp.save_state_dict(
             state_dict=A				,							storage_writer=dist_cp.FileSystemWriter(A     )				,							planner=DefaultSavePlanner()				,							)
         logger.info(F"""Model saved to {ckpt_dir}"""     )






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A				,							A=0     ):




   '''simple docstring'''

   accelerator.wait_for_everyone()
   with FSDP.state_dict_type(
       A				,							fsdp_plugin.state_dict_type				,							fsdp_plugin.state_dict_config				,							fsdp_plugin.optim_state_dict_config     ):
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         if type(A     ) != FSDP and accelerator.process_index != 0:
            if not fsdp_plugin.sync_module_states:
               raise ValueError(
                   "Set the `sync_module_states` flag to `True` so that model states are synced across processes when "
                   "initializing FSDP object"     )
            return
         UpperCAmelCase__						=F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
         UpperCAmelCase__						=os.path.join(A				,							A     )
         logger.info(F"""Loading model from {input_model_file}"""     )
         UpperCAmelCase__						=torch.load(A     )
         logger.info(F"""Model loaded from {input_model_file}"""     )
      elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
         UpperCAmelCase__						=(
             F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
             if model_index == 0
             else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
         )
         UpperCAmelCase__						=os.path.join(A				,							A     )
         logger.info(F"""Loading model from {input_model_file}"""     )
         UpperCAmelCase__						=torch.load(A     )
         logger.info(F"""Model loaded from {input_model_file}"""     )
      elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
         UpperCAmelCase__						=(
             os.path.join(A				,							F"""{MODEL_NAME}_{model_index}"""     )
             if F"""{MODEL_NAME}""" not in input_dir
             else input_dir
         )
         logger.info(F"""Loading model from {ckpt_dir}"""     )
         UpperCAmelCase__						={"model": model.state_dict()}
         dist_cp.load_state_dict(
             state_dict=A				,							storage_reader=dist_cp.FileSystemReader(A     )				,							planner=DefaultLoadPlanner()				,							)
         UpperCAmelCase__						=state_dict["model"]
         logger.info(F"""Model loaded from {ckpt_dir}"""     )
      model.load_state_dict(A     )






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A				,							A				,							A=0     ):




   '''simple docstring'''

   os.makedirs(A				,							exist_ok=A     )
   with FSDP.state_dict_type(
       A				,							fsdp_plugin.state_dict_type				,							fsdp_plugin.state_dict_config				,							fsdp_plugin.optim_state_dict_config     ):
      UpperCAmelCase__						=FSDP.optim_state_dict(A				,							A     )
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         if accelerator.process_index == 0:
            UpperCAmelCase__						=(
                F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
            )
            UpperCAmelCase__						=os.path.join(A				,							A     )
            logger.info(F"""Saving Optimizer state to {output_optimizer_file}"""     )
            torch.save(A				,							A     )
            logger.info(F"""Optimizer state saved in {output_optimizer_file}"""     )
      else:
         UpperCAmelCase__						=os.path.join(A				,							F"""{OPTIMIZER_NAME}_{optimizer_index}"""     )
         os.makedirs(A				,							exist_ok=A     )
         logger.info(F"""Saving Optimizer state to {ckpt_dir}"""     )
         dist_cp.save_state_dict(
             state_dict={"optimizer": optim_state}				,							storage_writer=dist_cp.FileSystemWriter(A     )				,							planner=DefaultSavePlanner()				,							)
         logger.info(F"""Optimizer state saved in {ckpt_dir}"""     )






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A				,							A				,							A=0     ):




   '''simple docstring'''

   accelerator.wait_for_everyone()
   with FSDP.state_dict_type(
       A				,							fsdp_plugin.state_dict_type				,							fsdp_plugin.state_dict_config				,							fsdp_plugin.optim_state_dict_config     ):
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         UpperCAmelCase__						=None
         # below check should work but currently it isn't working (mostly opytorch issue),
         # in the meantime disabling it at the cost of excess memory usage
         # if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
         UpperCAmelCase__						=(
             F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
         )
         UpperCAmelCase__						=os.path.join(A				,							A     )
         logger.info(F"""Loading Optimizer state from {input_optimizer_file}"""     )
         UpperCAmelCase__						=torch.load(A     )
         logger.info(F"""Optimizer state loaded from {input_optimizer_file}"""     )
      else:
         UpperCAmelCase__						=(
             os.path.join(A				,							F"""{OPTIMIZER_NAME}_{optimizer_index}"""     )
             if F"""{OPTIMIZER_NAME}""" not in input_dir
             else input_dir
         )
         logger.info(F"""Loading Optimizer from {ckpt_dir}"""     )
         UpperCAmelCase__						=load_sharded_optimizer_state_dict(
             model_state_dict=model.state_dict()				,							optimizer_key="optimizer"				,							storage_reader=dist_cp.FileSystemReader(A     )				,							)
         UpperCAmelCase__						=optim_state["optimizer"]
         logger.info(F"""Optimizer loaded from {ckpt_dir}"""     )
      UpperCAmelCase__						=FSDP.optim_state_dict_to_load(A				,							A				,							A     )
      optimizer.load_state_dict(A     )



