parameters: (bert_config, input_tensor, output_weights, positions,
                           label_ids, label_weights)
return_statement: return (loss, per_example_loss, log_probs)
docstring: Get loss and log probs for the masked LM.
docstring_summary: Get loss and log probs for the masked LM.
docstring_tokens: Get loss and log probs for the masked LM .
function: def get_masked_lm_output(bert_config, input_tensor, output_weights, positions,
                           label_ids, label_weights):
    """Get loss and log probs for the masked LM."""
    input_tensor = gather_indexes(input_tensor, positions)
  
    with tf.variable_scope("cls/predictions"):
      # We apply one more non-linear transformation before the output layer.
      # This matrix is not used after pre-training.
      with tf.variable_scope("transform"):
        input_tensor = tf.layers.dense(
            input_tensor,
            units=bert_config.hidden_size,
            activation=modeling.get_activation(bert_config.hidden_act),
            kernel_initializer=modeling.create_initializer(
                bert_config.initializer_range))
        input_tensor = modeling.layer_norm(input_tensor)
  
      # The output weights are the same as the input embeddings, but there is
      # an output-only bias for each token.
      output_bias = tf.get_variable(
          "output_bias",
          shape=[bert_config.vocab_size],
          initializer=tf.zeros_initializer())
      logits = tf.matmul(input_tensor, output_weights, transpose_b=True)
      logits = tf.nn.bias_add(logits, output_bias)
      log_probs = tf.nn.log_softmax(logits, axis=-1)
  
      label_ids = tf.reshape(label_ids, [-1])
      label_weights = tf.reshape(label_weights, [-1])
  
      one_hot_labels = tf.one_hot(
          label_ids, depth=bert_config.vocab_size, dtype=tf.float32)
  
      # The `positions` tensor might be zero-padded (if the sequence is too
      # short to have the maximum number of predictions). The `label_weights`
      # tensor has a value of 1.0 for every real prediction and 0.0 for the
      # padding predictions.
      per_example_loss = -tf.reduce_sum(log_probs * one_hot_labels, axis=[-1])
      numerator = tf.reduce_sum(label_weights * per_example_loss)
      denominator = tf.reduce_sum(label_weights) + 1e-5
      loss = numerator / denominator
  
    return (loss, per_example_loss, log_probs)
function_tokens: def get_masked_lm_output ( bert_config , input_tensor , output_weights , positions , label_ids , label_weights ) : input_tensor = gather_indexes ( input_tensor , positions ) with tf . variable_scope ( "cls/predictions" ) : # We apply one more non-linear transformation before the output layer. # This matrix is not used after pre-training. with tf . variable_scope ( "transform" ) : input_tensor = tf . layers . dense ( input_tensor , units = bert_config . hidden_size , activation = modeling . get_activation ( bert_config . hidden_act ) , kernel_initializer = modeling . create_initializer ( bert_config . initializer_range ) ) input_tensor = modeling . layer_norm ( input_tensor ) # The output weights are the same as the input embeddings, but there is # an output-only bias for each token. output_bias = tf . get_variable ( "output_bias" , shape = [ bert_config . vocab_size ] , initializer = tf . zeros_initializer ( ) ) logits = tf . matmul ( input_tensor , output_weights , transpose_b = True ) logits = tf . nn . bias_add ( logits , output_bias ) log_probs = tf . nn . log_softmax ( logits , axis = - 1 ) label_ids = tf . reshape ( label_ids , [ - 1 ] ) label_weights = tf . reshape ( label_weights , [ - 1 ] ) one_hot_labels = tf . one_hot ( label_ids , depth = bert_config . vocab_size , dtype = tf . float32 ) # The `positions` tensor might be zero-padded (if the sequence is too # short to have the maximum number of predictions). The `label_weights` # tensor has a value of 1.0 for every real prediction and 0.0 for the # padding predictions. per_example_loss = - tf . reduce_sum ( log_probs * one_hot_labels , axis = [ - 1 ] ) numerator = tf . reduce_sum ( label_weights * per_example_loss ) denominator = tf . reduce_sum ( label_weights ) + 1e-5 loss = numerator / denominator return ( loss , per_example_loss , log_probs )