parameters: (self, model)
return_statement: return distributed_model
docstring: Return distributed data parallel model (Only work in dygraph mode)
  
          Args:
              model (Layer): the user-defind model which inherits Layer.
  
          Returns:
              distributed data parallel model which inherits Layer.
  
          Examples:
  
              .. code-block:: python
  
                  import paddle
                  import paddle.nn as nn
                  from paddle.distributed import fleet
  
                  class LinearNet(nn.Layer):
                      def __init__(self):
                          super(LinearNet, self).__init__()
                          self._linear1 = nn.Linear(10, 10)
                          self._linear2 = nn.Linear(10, 1)
  
                      def forward(self, x):
                          return self._linear2(self._linear1(x))
  
                  # 1. initialize fleet environment
                  fleet.init(is_collective=True)
  
                  # 2. create layer & optimizer
                  layer = LinearNet()
                  loss_fn = nn.MSELoss()
                  adam = paddle.optimizer.Adam(
                      learning_rate=0.001, parameters=layer.parameters())
  
                  # 3. get data_parallel model using fleet
                  adam = fleet.distributed_optimizer(adam)
                  dp_layer = fleet.distributed_model(layer)
  
                  # 4. run layer
                  inputs = paddle.randn([10, 10], 'float32')
                  outputs = dp_layer(inputs)
                  labels = paddle.randn([10, 1], 'float32')
                  loss = loss_fn(outputs, labels)
  
                  print("loss:", loss.numpy())
  
                  loss.backward()
  
                  adam.step()
                  adam.clear_grad()
docstring_summary: Return distributed data parallel model (Only work in dygraph mode)
docstring_tokens: Return distributed data parallel model ( Only work in dygraph mode )
function: def distributed_model(self, model):
          """
          Return distributed data parallel model (Only work in dygraph mode)
  
          Args:
              model (Layer): the user-defind model which inherits Layer.
  
          Returns:
              distributed data parallel model which inherits Layer.
  
          Examples:
  
              .. code-block:: python
  
                  import paddle
                  import paddle.nn as nn
                  from paddle.distributed import fleet
  
                  class LinearNet(nn.Layer):
                      def __init__(self):
                          super(LinearNet, self).__init__()
                          self._linear1 = nn.Linear(10, 10)
                          self._linear2 = nn.Linear(10, 1)
  
                      def forward(self, x):
                          return self._linear2(self._linear1(x))
  
                  # 1. initialize fleet environment
                  fleet.init(is_collective=True)
  
                  # 2. create layer & optimizer
                  layer = LinearNet()
                  loss_fn = nn.MSELoss()
                  adam = paddle.optimizer.Adam(
                      learning_rate=0.001, parameters=layer.parameters())
  
                  # 3. get data_parallel model using fleet
                  adam = fleet.distributed_optimizer(adam)
                  dp_layer = fleet.distributed_model(layer)
  
                  # 4. run layer
                  inputs = paddle.randn([10, 10], 'float32')
                  outputs = dp_layer(inputs)
                  labels = paddle.randn([10, 1], 'float32')
                  loss = loss_fn(outputs, labels)
  
                  print("loss:", loss.numpy())
  
                  loss.backward()
  
                  adam.step()
                  adam.clear_grad()
  
  
          """
          assert model is not None, "model should not be None"
          if self.worker_num() <= 1:
              return model
  
          if self._user_defined_strategy.heter_ccl_mode == True:
              distributed_model = paddle.DataParallel(
                  model,
                  comm_buffer_size=self._user_defined_strategy.
                  fuse_grad_size_in_MB,
                  last_comm_buffer_size=self._user_defined_strategy.
                  last_comm_group_size_MB,
                  find_unused_parameters=self._user_defined_strategy.
                  find_unused_parameters)
              return distributed_model
  
          if self._hcg.get_parallel_mode() == ParallelMode.SHARDING_PARALLEL:
              distributed_model = ShardingParallel(
                  model, self._hcg, strategy=self._user_defined_strategy)
          elif self._hcg.get_parallel_mode() == ParallelMode.DATA_PARALLEL:
  
              # NOTE (JZ-LIANG) init parameters broadcast within sharding group
              # normally it should be done inside DataParallel
              if self.sharding_degree > 1:
                  from paddle.distributed.fleet.utils.hybrid_parallel_util import broadcast_mp_parameters, broadcast_sharding_parameters
                  assert self.sharding_degree == self._hcg.get_sharding_parallel_world_size(
                  )
                  broadcast_sharding_parameters(model, self._hcg)
              distributed_model = paddle.DataParallel(
                  model,
                  comm_buffer_size=self._user_defined_strategy.
                  fuse_grad_size_in_MB,
                  last_comm_buffer_size=self._user_defined_strategy.
                  last_comm_group_size_MB,
                  find_unused_parameters=self._user_defined_strategy.
                  find_unused_parameters)
          elif self._hcg.get_parallel_mode() == ParallelMode.TENSOR_PARALLEL:
              distributed_model = TensorParallel(
                  model, self._hcg, strategy=self._user_defined_strategy)
          elif self._hcg.get_parallel_mode() == ParallelMode.PIPELINE_PARALLEL:
              distributed_model = PipelineParallel(
                  model, self._hcg, strategy=self._user_defined_strategy)
  
          return distributed_model
function_tokens: def distributed_model ( self , model ) : assert model is not None , "model should not be None" if self . worker_num ( ) <= 1 : return model if self . _user_defined_strategy . heter_ccl_mode == True : distributed_model = paddle . DataParallel ( model , comm_buffer_size = self . _user_defined_strategy . fuse_grad_size_in_MB , last_comm_buffer_size = self . _user_defined_strategy . last_comm_group_size_MB , find_unused_parameters = self . _user_defined_strategy . find_unused_parameters ) return distributed_model if self . _hcg . get_parallel_mode ( ) == ParallelMode . SHARDING_PARALLEL : distributed_model = ShardingParallel ( model , self . _hcg , strategy = self . _user_defined_strategy ) elif self . _hcg . get_parallel_mode ( ) == ParallelMode . DATA_PARALLEL : # NOTE (JZ-LIANG) init parameters broadcast within sharding group # normally it should be done inside DataParallel if self . sharding_degree > 1 : from paddle . distributed . fleet . utils . hybrid_parallel_util import broadcast_mp_parameters , broadcast_sharding_parameters assert self . sharding_degree == self . _hcg . get_sharding_parallel_world_size ( ) broadcast_sharding_parameters ( model , self . _hcg ) distributed_model = paddle . DataParallel ( model , comm_buffer_size = self . _user_defined_strategy . fuse_grad_size_in_MB , last_comm_buffer_size = self . _user_defined_strategy . last_comm_group_size_MB , find_unused_parameters = self . _user_defined_strategy . find_unused_parameters ) elif self . _hcg . get_parallel_mode ( ) == ParallelMode . TENSOR_PARALLEL : distributed_model = TensorParallel ( model , self . _hcg , strategy = self . _user_defined_strategy ) elif self . _hcg . get_parallel_mode ( ) == ParallelMode . PIPELINE_PARALLEL : distributed_model = PipelineParallel ( model , self . _hcg , strategy = self . _user_defined_strategy ) return distributed_model