parameters: (self)
docstring: Create kvstore.
docstring_summary: Create kvstore.
docstring_tokens: Create kvstore .
function: def _init_kvstore(self):
          """Create kvstore."""
          config = self._kvstore_params
          # configure kvstore, update_on_kvstore and self._distributed on three cases:
          if self._contains_sparse_weight:
              # If weight is sparse, kvstore must be present and the weight must be updated on kvstore.
              # The training loop is the following:
              #    - row_sparse_pull(sparse_weight)
              #    - forward()
              #    - backward()
              #    - push_and_update(grad)
              #    - pull(weight)
              kvstore, update_on_kvstore = _create_sparse_kvstore(config['kvstore'])
              self._distributed = 'dist' in kvstore.type
              # raise err if user provides unsupported configs
              if config['update_on_kvstore'] is False:
                  raise ValueError("Cannot set update_on_kvstore=False when sparse weights "
                                   "are present.")
  
          elif self._contains_sparse_grad:
              # For single node training with dense weight and sparse grad,
              # we prefer update_on_kvstore=False because this is usually faster.
              # This means we push and pull sparse gradients, and we do not store weight in kvstore.
              # The training loop is the following:
              #    - forward()
              #    - backward()
              #    - push(grad)
              #    - pull(grad)
              #    - update(grad, weight)
              #
              # For multi-node training with dense weight and sparse grad,
              # only update_on_kvstore=True is supported, due to the fact that
              # kv.row_sparse_pull(grad) is not implemented.
              # Therefore, we push sparse gradients and pull dense weights.
              # The training loop contains:
              #    - forward()
              #    - backward()
              #    - push_and_update(grad)
              #    - pull(weight)
              arg_arrays = {param._uuid: param.data(self._devices[0]) for param in self._params}
              kvstore, _ = _create_kvstore(config['kvstore'], len(self._devices), arg_arrays)
              self._distributed = 'dist' in kvstore.type if kvstore else False
              update_on_kvstore = self._distributed
              # raise err if user provides unsupported configs
              if config['update_on_kvstore'] is not None:
                  if config['update_on_kvstore'] is False and self._distributed:
                      raise ValueError("Cannot set update_on_kvstore=False on dist kvstore "
                                       "when sparse gradients are present.")
                  update_on_kvstore = config['update_on_kvstore']
              # raise err if a custom kvstore is used for sparse training
              if kvstore is not None and not isinstance(kvstore, KVStore):
                  raise ValueError("Cannot use {} for multi-device training with sparse gradients"
                                   .format(type(kvstore)))
  
          else:
              # Training with dense weight and dense gradients.
              # The only unsupported mode is async with update_on_kvstore=False
              arg_arrays = {param._uuid: param.data(self._devices[0]) for param in self._params}
              kvstore, update_on_kvstore = _create_kvstore(config['kvstore'], len(self._devices),
                                                           arg_arrays)
              self._distributed = 'dist' in kvstore.type if kvstore else False
              if self._distributed and 'async' in kvstore.type:
                  update_on_kvstore = True
                  # raise err if user provides unsupported configs
                  if config['update_on_kvstore'] is False:
                      raise ValueError("Please set update_on_kvstore=True "
                                       "when training in async mode.")
              if config['update_on_kvstore'] is not None:
                  update_on_kvstore = config['update_on_kvstore']
              # raise err if update_on_kvstore is set to True with kvstores that do not support optimizers
              if update_on_kvstore and not kvstore.is_capable('optimizer'):
                  if config['update_on_kvstore']:
                      raise ValueError("Please set update_on_kvstore=False "
                                       "when training with {}".format(type(kvstore)))
                  update_on_kvstore = False
  
          # set grad compression and optimizers
          if kvstore:
              if self._compression_params:
                  kvstore.set_gradient_compression(self._compression_params)
              if update_on_kvstore:
                  # optimizer preferably needs to be set before init for multiprecision
                  kvstore.set_optimizer(self._optimizer)
              self._kvstore = kvstore
              self._update_on_kvstore = update_on_kvstore
          else:
              self._kvstore = None
              self._update_on_kvstore = None
  
          self._kv_initialized = True
function_tokens: def _init_kvstore ( self ) : config = self . _kvstore_params # configure kvstore, update_on_kvstore and self._distributed on three cases: if self . _contains_sparse_weight : # If weight is sparse, kvstore must be present and the weight must be updated on kvstore. # The training loop is the following: #    - row_sparse_pull(sparse_weight) #    - forward() #    - backward() #    - push_and_update(grad) #    - pull(weight) kvstore , update_on_kvstore = _create_sparse_kvstore ( config [ 'kvstore' ] ) self . _distributed = 'dist' in kvstore . type # raise err if user provides unsupported configs if config [ 'update_on_kvstore' ] is False : raise ValueError ( "Cannot set update_on_kvstore=False when sparse weights " "are present." ) elif self . _contains_sparse_grad : # For single node training with dense weight and sparse grad, # we prefer update_on_kvstore=False because this is usually faster. # This means we push and pull sparse gradients, and we do not store weight in kvstore. # The training loop is the following: #    - forward() #    - backward() #    - push(grad) #    - pull(grad) #    - update(grad, weight) # # For multi-node training with dense weight and sparse grad, # only update_on_kvstore=True is supported, due to the fact that # kv.row_sparse_pull(grad) is not implemented. # Therefore, we push sparse gradients and pull dense weights. # The training loop contains: #    - forward() #    - backward() #    - push_and_update(grad) #    - pull(weight) arg_arrays = { param . _uuid : param . data ( self . _devices [ 0 ] ) for param in self . _params } kvstore , _ = _create_kvstore ( config [ 'kvstore' ] , len ( self . _devices ) , arg_arrays ) self . _distributed = 'dist' in kvstore . type if kvstore else False update_on_kvstore = self . _distributed # raise err if user provides unsupported configs if config [ 'update_on_kvstore' ] is not None : if config [ 'update_on_kvstore' ] is False and self . _distributed : raise ValueError ( "Cannot set update_on_kvstore=False on dist kvstore " "when sparse gradients are present." ) update_on_kvstore = config [ 'update_on_kvstore' ] # raise err if a custom kvstore is used for sparse training if kvstore is not None and not isinstance ( kvstore , KVStore ) : raise ValueError ( "Cannot use {} for multi-device training with sparse gradients" . format ( type ( kvstore ) ) ) else : # Training with dense weight and dense gradients. # The only unsupported mode is async with update_on_kvstore=False arg_arrays = { param . _uuid : param . data ( self . _devices [ 0 ] ) for param in self . _params } kvstore , update_on_kvstore = _create_kvstore ( config [ 'kvstore' ] , len ( self . _devices ) , arg_arrays ) self . _distributed = 'dist' in kvstore . type if kvstore else False if self . _distributed and 'async' in kvstore . type : update_on_kvstore = True # raise err if user provides unsupported configs if config [ 'update_on_kvstore' ] is False : raise ValueError ( "Please set update_on_kvstore=True " "when training in async mode." ) if config [ 'update_on_kvstore' ] is not None : update_on_kvstore = config [ 'update_on_kvstore' ] # raise err if update_on_kvstore is set to True with kvstores that do not support optimizers if update_on_kvstore and not kvstore . is_capable ( 'optimizer' ) : if config [ 'update_on_kvstore' ] : raise ValueError ( "Please set update_on_kvstore=False " "when training with {}" . format ( type ( kvstore ) ) ) update_on_kvstore = False # set grad compression and optimizers if kvstore : if self . _compression_params : kvstore . set_gradient_compression ( self . _compression_params ) if update_on_kvstore : # optimizer preferably needs to be set before init for multiprecision kvstore . set_optimizer ( self . _optimizer ) self . _kvstore = kvstore self . _update_on_kvstore = update_on_kvstore else : self . _kvstore = None self . _update_on_kvstore = None self . _kv_initialized = True