





import numpy as np
import torch
from torch.utils.data import Dataset, IterableDataset

from ..utils.generic import ModelOutput

class     snake_case_		(   a							):






 '''simple docstring'''





 def __init__(  self,		A_,		A_,		A_		)	->       str:
    UpperCAmelCase__						=dataset
    UpperCAmelCase__						=process
    UpperCAmelCase__						=params

 def __len__(  self		)	->       Any:
    return len(self.dataset		)






 def __getitem__(  self,		A_		)	->       Optional[Any]:
    UpperCAmelCase__						=self.dataset[i]
    UpperCAmelCase__						=self.process(A_,		**self.params		)
    return processed

class     snake_case_		(   a							):






 '''simple docstring'''





 def __init__(  self,		A_,		A_,		A_,		A_=None		)	->       Dict:
    UpperCAmelCase__						=loader
    UpperCAmelCase__						=infer
    UpperCAmelCase__						=params
    if loader_batch_size == 1:
       # Let's spare some time by deactivating altogether
       UpperCAmelCase__						=None
    UpperCAmelCase__						=loader_batch_size

    # Internal bookkeeping
    UpperCAmelCase__						=None
    UpperCAmelCase__						=None

 def __len__(  self		)	->       Dict:
    return len(self.loader		)

 def __iter__(  self		)	->       Any:
    UpperCAmelCase__						=iter(self.loader		)
    return self

 def  __UpperCAmelCase						(  self		)	->       Optional[Any]:
    if isinstance(self._loader_batch_data,		torch.Tensor		):
       # Batch data is simple tensor, just fetch the slice
       UpperCAmelCase__						=self._loader_batch_data[self._loader_batch_index]
    else:
       # Batch data is assumed to be BaseModelOutput (or dict)
       UpperCAmelCase__						={}
       for k, element in self._loader_batch_data.items():
          if isinstance(A_,		A_		):
             # Convert ModelOutput to tuple first
             UpperCAmelCase__						=element.to_tuple()
             if isinstance(element[0],		torch.Tensor		):
                UpperCAmelCase__						=tuple(el[self._loader_batch_index].unsqueeze(0		) for el in element		)
             elif isinstance(element[0],		np.ndarray		):
                UpperCAmelCase__						=tuple(np.expand_dims(el[self._loader_batch_index],		0		) for el in element		)
             continue
          if k in {"hidden_states", "past_key_values", "attentions"} and isinstance(A_,		A_		):
             # Those are stored as lists of tensors so need specific unbatching.
             if isinstance(element[0],		torch.Tensor		):
                UpperCAmelCase__						=tuple(el[self._loader_batch_index].unsqueeze(0		) for el in element		)
             elif isinstance(element[0],		np.ndarray		):
                UpperCAmelCase__						=tuple(np.expand_dims(el[self._loader_batch_index],		0		) for el in element		)
             continue
          if element is None:
             # This can happen for optional data that get passed around
             UpperCAmelCase__						=None
          elif isinstance(element[self._loader_batch_index],		torch.Tensor		):
             # Take correct batch data, but make it looked like batch_size=1
             # For compatibility with other methods within transformers

             UpperCAmelCase__						=element[self._loader_batch_index].unsqueeze(0		)
          elif isinstance(element[self._loader_batch_index],		np.ndarray		):
             # Take correct batch data, but make it looked like batch_size=1
             # For compatibility with other methods within transformers
             UpperCAmelCase__						=np.expand_dims(element[self._loader_batch_index],		0		)
          else:
             # This is typically a list, so no need to `unsqueeze`.
             UpperCAmelCase__						=element[self._loader_batch_index]
            # Recreate the element by reusing the original class to make it look
            # batch_size=1
       UpperCAmelCase__						=self._loader_batch_data.__class__(A_		)
    self._loader_batch_index += 1
    return result






 def  __UpperCAmelCase						(  self		)	->       int:
    if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
       # We are currently unrolling a batch so we just need to return
       # the current item within a batch
       return self.loader_batch_item()

    # We're out of items within a batch
    UpperCAmelCase__						=next(self.iterator		)
    UpperCAmelCase__						=self.infer(A_,		**self.params		)
    # We now have a batch of "inferred things".
    if self.loader_batch_size is not None:
       # Try to infer the size of the batch
       if isinstance(A_,		torch.Tensor		):
          UpperCAmelCase__						=processed
       else:
          UpperCAmelCase__						=list(processed.keys()		)[0]
          UpperCAmelCase__						=processed[key]
       if isinstance(A_,		A_		):
          UpperCAmelCase__						=len(A_		)
       else:
          UpperCAmelCase__						=first_tensor.shape[0]
       if 0 < observed_batch_size < self.loader_batch_size:
          # could be last batch so we can't unroll as many
          # elements.
          UpperCAmelCase__						=observed_batch_size
       # Setting internal index to unwrap the batch
       UpperCAmelCase__						=processed
       UpperCAmelCase__						=0
       return self.loader_batch_item()
    else:
       # We're not unrolling batches
       return processed

class     snake_case_		(   a							):






 '''simple docstring'''





 def __init__(  self,		A_,		A_,		A_,		A_=None		)	->       List[Any]:
    super().__init__(A_,		A_,		A_		)

 def __iter__(  self		)	->       Tuple:
    UpperCAmelCase__						=iter(self.loader		)
    UpperCAmelCase__						=None
    return self






 def  __UpperCAmelCase						(  self		)	->       List[Any]:
    if self.subiterator is None:
       UpperCAmelCase__						=self.infer(next(self.iterator		),		**self.params		)
    try:
       # Try to return next item
       UpperCAmelCase__						=next(self.subiterator		)
    except StopIteration:
       # When a preprocess iterator ends, we can start lookig at the next item
       # ChunkIterator will keep feeding until ALL elements of iterator
       # all have created their subiterator and have been iterating against.
       #
       # Another way to look at it, is we're basically flattening lists of lists
       # into a single list, but with generators
       UpperCAmelCase__						=self.infer(next(self.iterator		),		**self.params		)
       UpperCAmelCase__						=next(self.subiterator		)
    return processed

class     snake_case_		(   a							):






 '''simple docstring'''





 def __iter__(  self		)	->       str:
    UpperCAmelCase__						=iter(self.loader		)
    return self






 def  __UpperCAmelCase						(  self		)	->       Any:
    # Extremely similar to PipelineIterator in its unpacking mechanism
    # BUT, we have an extra required item which is the presence of `is_last`
    # That is because everything is flattened by `PipelineChunkIterator` we
    # need to keep track of how to regroup here in the original `process`
    # boundaries so that `process` and `postprocess` see the same data.

    # This iterator accumulates items (possibly while unbatching) until it
    # its a `is_last` and then just passes it on to the caller.
    UpperCAmelCase__						=False
    UpperCAmelCase__						=[]
    if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
       while self._loader_batch_index < self.loader_batch_size:
          UpperCAmelCase__						=self.loader_batch_item()
          UpperCAmelCase__						=item.pop("is_last"		)
          accumulator.append(A_		)
          if is_last:
             return accumulator

    while not is_last:
       UpperCAmelCase__						=self.infer(next(self.iterator		),		**self.params		)
       if self.loader_batch_size is not None:
          if isinstance(A_,		torch.Tensor		):
             UpperCAmelCase__						=processed
          else:
             UpperCAmelCase__						=list(processed.keys()		)[0]
             UpperCAmelCase__						=processed[key]
          if isinstance(A_,		A_		):
             UpperCAmelCase__						=len(A_		)
          else:
             UpperCAmelCase__						=first_tensor.shape[0]
          if 0 < observed_batch_size < self.loader_batch_size:
             # could be last batch so we can't unroll as many
             # elements.
             UpperCAmelCase__						=observed_batch_size
          UpperCAmelCase__						=processed
          UpperCAmelCase__						=0
          while self._loader_batch_index < self.loader_batch_size:
             UpperCAmelCase__						=self.loader_batch_item()
             UpperCAmelCase__						=item.pop("is_last"		)
             accumulator.append(A_		)
             if is_last:
                return accumulator
       else:
          UpperCAmelCase__						=processed
          UpperCAmelCase__						=item.pop("is_last"		)
          accumulator.append(A_		)
    return accumulator

class     snake_case_		(   a							):






 '''simple docstring'''





 def __init__(  self,		A_,		A_		)	->       List[Any]:
    UpperCAmelCase__						=dataset
    UpperCAmelCase__						=key

 def __len__(  self		)	->       int:
    return len(self.dataset		)






 def __getitem__(  self,		A_		)	->       Optional[int]:
    return self.dataset[i][self.key]

class     snake_case_		(   a							):






 '''simple docstring'''





 def __init__(  self,		A_,		A_,		A_		)	->       Any:
    UpperCAmelCase__						=dataset
    UpperCAmelCase__						=keya
    UpperCAmelCase__						=keya

 def __len__(  self		)	->       Tuple:
    return len(self.dataset		)






 def __getitem__(  self,		A_		)	->       str:
    return {"text": self.dataset[i][self.keya], "text_pair": self.dataset[i][self.keya]}










import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version('>=', FSDP_PYTORCH_VERSION):
    import torch.distributed.checkpoint as dist_cp
    from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
    from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
    from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
    from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


UpperCamelCase_												=	get_logger(__name__)






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A				,							A=0     ):




   '''simple docstring'''

   os.makedirs(A				,							exist_ok=A     )
   with FSDP.state_dict_type(
       A				,							fsdp_plugin.state_dict_type				,							fsdp_plugin.state_dict_config				,							fsdp_plugin.optim_state_dict_config     ):
      UpperCAmelCase__						=model.state_dict()
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         UpperCAmelCase__						=F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
         UpperCAmelCase__						=os.path.join(A				,							A     )
         if accelerator.process_index == 0:
            logger.info(F"""Saving model to {output_model_file}"""     )
            torch.save(A				,							A     )
            logger.info(F"""Model saved to {output_model_file}"""     )
      elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
         UpperCAmelCase__						=(
             F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
             if model_index == 0
             else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
         )
         UpperCAmelCase__						=os.path.join(A				,							A     )
         logger.info(F"""Saving model to {output_model_file}"""     )
         torch.save(A				,							A     )
         logger.info(F"""Model saved to {output_model_file}"""     )
      elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
         UpperCAmelCase__						=os.path.join(A				,							F"""{MODEL_NAME}_{model_index}"""     )
         os.makedirs(A				,							exist_ok=A     )
         logger.info(F"""Saving model to {ckpt_dir}"""     )
         UpperCAmelCase__						={"model": state_dict}

         dist_cp.save_state_dict(
             state_dict=A				,							storage_writer=dist_cp.FileSystemWriter(A     )				,							planner=DefaultSavePlanner()				,							)
         logger.info(F"""Model saved to {ckpt_dir}"""     )






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A				,							A=0     ):




   '''simple docstring'''

   accelerator.wait_for_everyone()
   with FSDP.state_dict_type(
       A				,							fsdp_plugin.state_dict_type				,							fsdp_plugin.state_dict_config				,							fsdp_plugin.optim_state_dict_config     ):
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         if type(A     ) != FSDP and accelerator.process_index != 0:
            if not fsdp_plugin.sync_module_states:
               raise ValueError(
                   "Set the `sync_module_states` flag to `True` so that model states are synced across processes when "
                   "initializing FSDP object"     )
            return
         UpperCAmelCase__						=F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
         UpperCAmelCase__						=os.path.join(A				,							A     )
         logger.info(F"""Loading model from {input_model_file}"""     )
         UpperCAmelCase__						=torch.load(A     )
         logger.info(F"""Model loaded from {input_model_file}"""     )
      elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
         UpperCAmelCase__						=(
             F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
             if model_index == 0
             else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
         )
         UpperCAmelCase__						=os.path.join(A				,							A     )
         logger.info(F"""Loading model from {input_model_file}"""     )
         UpperCAmelCase__						=torch.load(A     )
         logger.info(F"""Model loaded from {input_model_file}"""     )
      elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
         UpperCAmelCase__						=(
             os.path.join(A				,							F"""{MODEL_NAME}_{model_index}"""     )
             if F"""{MODEL_NAME}""" not in input_dir
             else input_dir
         )
         logger.info(F"""Loading model from {ckpt_dir}"""     )
         UpperCAmelCase__						={"model": model.state_dict()}
         dist_cp.load_state_dict(
             state_dict=A				,							storage_reader=dist_cp.FileSystemReader(A     )				,							planner=DefaultLoadPlanner()				,							)
         UpperCAmelCase__						=state_dict["model"]
         logger.info(F"""Model loaded from {ckpt_dir}"""     )
      model.load_state_dict(A     )






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A				,							A				,							A=0     ):




   '''simple docstring'''

   os.makedirs(A				,							exist_ok=A     )
   with FSDP.state_dict_type(
       A				,							fsdp_plugin.state_dict_type				,							fsdp_plugin.state_dict_config				,							fsdp_plugin.optim_state_dict_config     ):
      UpperCAmelCase__						=FSDP.optim_state_dict(A				,							A     )
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         if accelerator.process_index == 0:
            UpperCAmelCase__						=(
                F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
            )
            UpperCAmelCase__						=os.path.join(A				,							A     )
            logger.info(F"""Saving Optimizer state to {output_optimizer_file}"""     )
            torch.save(A				,							A     )
            logger.info(F"""Optimizer state saved in {output_optimizer_file}"""     )
      else:
         UpperCAmelCase__						=os.path.join(A				,							F"""{OPTIMIZER_NAME}_{optimizer_index}"""     )
         os.makedirs(A				,							exist_ok=A     )
         logger.info(F"""Saving Optimizer state to {ckpt_dir}"""     )
         dist_cp.save_state_dict(
             state_dict={"optimizer": optim_state}				,							storage_writer=dist_cp.FileSystemWriter(A     )				,							planner=DefaultSavePlanner()				,							)
         logger.info(F"""Optimizer state saved in {ckpt_dir}"""     )






def 				_UpperCAmelCase			(    A				,							A				,							A				,							A				,							A				,							A=0     ):




   '''simple docstring'''

   accelerator.wait_for_everyone()
   with FSDP.state_dict_type(
       A				,							fsdp_plugin.state_dict_type				,							fsdp_plugin.state_dict_config				,							fsdp_plugin.optim_state_dict_config     ):
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         UpperCAmelCase__						=None
         # below check should work but currently it isn't working (mostly opytorch issue),
         # in the meantime disabling it at the cost of excess memory usage
         # if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
         UpperCAmelCase__						=(
             F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
         )
         UpperCAmelCase__						=os.path.join(A				,							A     )
         logger.info(F"""Loading Optimizer state from {input_optimizer_file}"""     )
         UpperCAmelCase__						=torch.load(A     )
         logger.info(F"""Optimizer state loaded from {input_optimizer_file}"""     )
      else:
         UpperCAmelCase__						=(
             os.path.join(A				,							F"""{OPTIMIZER_NAME}_{optimizer_index}"""     )
             if F"""{OPTIMIZER_NAME}""" not in input_dir
             else input_dir
         )
         logger.info(F"""Loading Optimizer from {ckpt_dir}"""     )
         UpperCAmelCase__						=load_sharded_optimizer_state_dict(
             model_state_dict=model.state_dict()				,							optimizer_key="optimizer"				,							storage_reader=dist_cp.FileSystemReader(A     )				,							)
         UpperCAmelCase__						=optim_state["optimizer"]
         logger.info(F"""Optimizer loaded from {ckpt_dir}"""     )
      UpperCAmelCase__						=FSDP.optim_state_dict_to_load(A				,							A				,							A     )
      optimizer.load_state_dict(A     )



