parameters: (self, optim: Type, *args, optim_params=None, **kwargs)
docstring: r"""
          Registers an optimizer with DDP such that the optimization for a
          parameter will run immediately when that parameter's gradient is
          finished with reduction, instead of waiting for all parameters'
          gradients to finish reduction. This can result in a training speedup
          depending on your workload since the optimizer can run while gradient
          reduction for other parameters are still ongoing. In addition, this has
          the potential to reduce peak memory consumption during training, as it
          only needs to load the per-parameter optimizer states of a single
          parameter at a time, instead of loading all per-parameter optimizer
          states at once.
  
          Args:
              optim_cls (Type): a ``torch.optim.Optimizer`` class to be registered
              as a fused optimizer.
              *args (Sequence[Any]): Arguments to forward to `optim_cls`.
              optim_params (Optional[Iterable[torch.Tensor]]): Set of parameters
              to optimize, similar to `params` argument of traditional `torch.optim`
              Optimizers. If this is omitted, all DDP model parameters will be
              optimized.
              **kwargs: (Dict[str, Any]): Keyword arguments to forward to `optim_cls`.
  
      .. warning ::
          _register_fused_optim should only be called once on a DDP instance,
          and registering multiple fused optimizers for the same DDP model
          is not currently supported. Please ping
          https://github.com/pytorch/pytorch/issues/71595 if this is necessary
          for your use case.
  
      .. warning ::
          _register_fused_optim and register_comm_hook currently do not
          compose together, meaning that custom DDP communication hooks are
          not supported with overlapped optimizers. Please ping
          https://github.com/pytorch/pytorch/issues/71595 if this is necessary
          for your use case.
  
      .. warning ::
          Gradient accumulation and DDP `no_sync` are currently not supported
          with overlapped optimizer. Please ping
          https://github.com/pytorch/pytorch/issues/71595 if this is necessary
          for your use case.
  
      Example::
  
          >>> torch.distributed.init_process_group(backend='nccl', world_size=4, init_method='...')
          >>> net = torch.nn.parallel.DistributedDataParallel(model, pg)
          >>> lr = 1e-2
          >>> betas = (0.9, 0.99)
          >>> eps = 1e-6
          >>> net._register_fused_optim(torch.optim.Adam, lr, betas=betas, eps=eps)
          >>> # Example with subset of parameters
          >>> params_to_opt = [list(net.parameters())[0]]
          >>> net._register_fused_optim(
              torch.optim.Adam, lr, optim_params=params_to_opt,  betas=betas, eps=eps
          )
docstring_summary: r"""
          Registers an optimizer with DDP such that the optimization for a
          parameter will run immediately when that parameter's gradient is
          finished with reduction, instead of waiting for all parameters'
          gradients to finish reduction. This can result in a training speedup
          depending on your workload since the optimizer can run while gradient
          reduction for other parameters are still ongoing. In addition, this has
          the potential to reduce peak memory consumption during training, as it
          only needs to load the per-parameter optimizer states of a single
          parameter at a time, instead of loading all per-parameter optimizer
          states at once.
docstring_tokens: r Registers an optimizer with DDP such that the optimization for a parameter will run immediately when that parameter s gradient is finished with reduction instead of waiting for all parameters gradients to finish reduction . This can result in a training speedup depending on your workload since the optimizer can run while gradient reduction for other parameters are still ongoing . In addition this has the potential to reduce peak memory consumption during training as it only needs to load the per - parameter optimizer states of a single parameter at a time instead of loading all per - parameter optimizer states at once .
function: def _register_fused_optim(self, optim: Type, *args, optim_params=None, **kwargs):
          r"""
          Registers an optimizer with DDP such that the optimization for a
          parameter will run immediately when that parameter's gradient is
          finished with reduction, instead of waiting for all parameters'
          gradients to finish reduction. This can result in a training speedup
          depending on your workload since the optimizer can run while gradient
          reduction for other parameters are still ongoing. In addition, this has
          the potential to reduce peak memory consumption during training, as it
          only needs to load the per-parameter optimizer states of a single
          parameter at a time, instead of loading all per-parameter optimizer
          states at once.
  
          Args:
              optim_cls (Type): a ``torch.optim.Optimizer`` class to be registered
              as a fused optimizer.
              *args (Sequence[Any]): Arguments to forward to `optim_cls`.
              optim_params (Optional[Iterable[torch.Tensor]]): Set of parameters
              to optimize, similar to `params` argument of traditional `torch.optim`
              Optimizers. If this is omitted, all DDP model parameters will be
              optimized.
              **kwargs: (Dict[str, Any]): Keyword arguments to forward to `optim_cls`.
  
      .. warning ::
          _register_fused_optim should only be called once on a DDP instance,
          and registering multiple fused optimizers for the same DDP model
          is not currently supported. Please ping
          https://github.com/pytorch/pytorch/issues/71595 if this is necessary
          for your use case.
  
      .. warning ::
          _register_fused_optim and register_comm_hook currently do not
          compose together, meaning that custom DDP communication hooks are
          not supported with overlapped optimizers. Please ping
          https://github.com/pytorch/pytorch/issues/71595 if this is necessary
          for your use case.
  
      .. warning ::
          Gradient accumulation and DDP `no_sync` are currently not supported
          with overlapped optimizer. Please ping
          https://github.com/pytorch/pytorch/issues/71595 if this is necessary
          for your use case.
  
      Example::
  
          >>> torch.distributed.init_process_group(backend='nccl', world_size=4, init_method='...')
          >>> net = torch.nn.parallel.DistributedDataParallel(model, pg)
          >>> lr = 1e-2
          >>> betas = (0.9, 0.99)
          >>> eps = 1e-6
          >>> net._register_fused_optim(torch.optim.Adam, lr, betas=betas, eps=eps)
          >>> # Example with subset of parameters
          >>> params_to_opt = [list(net.parameters())[0]]
          >>> net._register_fused_optim(
              torch.optim.Adam, lr, optim_params=params_to_opt,  betas=betas, eps=eps
          )
          """
          # Note: importing in function, otherwise this will cause a circular
          # import as optimizer_overlap module needs to import DistributedDataParallel.
          from torch.distributed.algorithms._optimizer_overlap import _as_overlapped_optim
  
          overlapped_optim = _as_overlapped_optim(optim, optim_params, *args, **kwargs)
          try:
              overlapped_optim.register_ddp(self)
          except NotImplementedError:
              raise RuntimeError(
                  f"{optim} does not support overlapped DDP. Please file an issue to PyTorch or the respective owner of {optim}."
              )
function_tokens: def _register_fused_optim ( self , optim : Type , * args , optim_params = None , * * kwargs ) : # Note: importing in function, otherwise this will cause a circular # import as optimizer_overlap module needs to import DistributedDataParallel. from torch . distributed . algorithms . _optimizer_overlap import _as_overlapped_optim overlapped_optim = _as_overlapped_optim ( optim , optim_params , * args , * * kwargs ) try : overlapped_optim . register_ddp ( self ) except NotImplementedError : raise RuntimeError ( f"{optim} does not support overlapped DDP. Please file an issue to PyTorch or the respective owner of {optim}." )