<commit_before>/* Copyright 2016 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#ifndef THIRD_PARTY_TENSORFLOW_CONTRIB_LINEAR_OPTIMIZER_KERNELS_LOGISTIC_LOSS_H_
#define THIRD_PARTY_TENSORFLOW_CONTRIB_LINEAR_OPTIMIZER_KERNELS_LOGISTIC_LOSS_H_

#include <algorithm>
#include <cmath>

#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"

namespace tensorflow {
struct logistic_loss {
  // Use an approximate step that is guaranteed to decrease the dual loss.
  // Derivation of this is available in  Page 14 Eq 16 of
  // http://arxiv.org/pdf/1211.2717v1.pdf
  inline static double ComputeUpdatedDual(const double label,
                                          const double example_weight,
                                          const double current_dual,
                                          const double wx,
                                          const double weighted_example_norm,
                                          const double primal_loss,
                                          const double dual_loss) {
    const double ywx = label * wx;
    // To avoid overflow, we compute derivative of logistic loss with respect to
    // log-odds as follows.
    double inverse_exp_term = 0;
    if (ywx > 0) {
      const double exp_minus_ywx = exp(-ywx);
      inverse_exp_term = exp_minus_ywx / (1 + exp_minus_ywx);
    } else {
      inverse_exp_term = 1 / (1 + exp(ywx));
    }
    // f(a) = sup (a*x  - f(x)) then a = f'(x), where a is the aproximate dual.
    const double approximate_dual = inverse_exp_term * label;
    const double delta_dual = approximate_dual - current_dual;
    // Upper bound on the smoothness constant of log loss. This is 0.25 i.e.
    // when log-odds is zero.
    const double gamma =
        (wx == 0) ? 0.25 : (1 - 2 * inverse_exp_term) / (2 * ywx);
    const double wx_dual = wx * current_dual * example_weight;
    const double delta_dual_squared = delta_dual * delta_dual;
    const double smooth_delta_dual_squared = delta_dual_squared * gamma * 0.5;
    double multiplier =
        (primal_loss + dual_loss + wx_dual + smooth_delta_dual_squared) /
        std::max(1.0,
                 delta_dual_squared *
                     (gamma +
                      weighted_example_norm * example_weight * example_weight));
    // Multiplier must be in the range [0, 1].
    multiplier = std::max(std::min(1.0, multiplier), 0.0);
    return current_dual + delta_dual * multiplier;
  }

  // Dual of logisitic loss function.
  // https://en.wikipedia.org/wiki/Convex_conjugate
  inline static double ComputeDualLoss(const double current_dual,
                                       const double example_label,
                                       const double example_weight) {
    // Dual of the logistic loss function is
    // ay * log(ay) + (1-ay) * log (1-ay), where a is the dual variable.<commit_msg>Refactor logistic loss into component parts. Change: 116471151<commit_after>/* Copyright 2016 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#ifndef THIRD_PARTY_TENSORFLOW_CONTRIB_LINEAR_OPTIMIZER_KERNELS_LOGISTIC_LOSS_H_
#define THIRD_PARTY_TENSORFLOW_CONTRIB_LINEAR_OPTIMIZER_KERNELS_LOGISTIC_LOSS_H_

#include <algorithm>
#include <cmath>

#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"

namespace tensorflow {
struct logistic_loss {
  // Partial derivative of the logistic loss w.r.t (1 + exp(-ywx)).
  inline static double PartialDerivativeLogisticLoss(const double wx,
                                                     const double label) {
    // To avoid overflow, we compute partial derivative of logistic loss as
    // follows.
    const double ywx = label * wx;
    if (ywx > 0) {
      const double exp_minus_ywx = exp(-ywx);
      return exp_minus_ywx / (1 + exp_minus_ywx);
    }
    return 1 / (1 + exp(ywx));
  }

  // Smoothness constant for the logistic loss.
  inline static double SmoothnessConstantLogisticLoss(
      const double partial_derivative_loss, const double wx,
      const double label) {
    // Upper bound on the smoothness constant of log loss. This is 0.25 i.e.
    // when log-odds is zero.
    return (wx == 0) ? 0.25
                     : (1 - 2 * partial_derivative_loss) / (2 * label * wx);
  }

  // Use an approximate step that is guaranteed to decrease the dual loss.
  // Derivation of this is available in  Page 14 Eq 16 of
  // http://arxiv.org/pdf/1211.2717v1.pdf
  inline static double ComputeUpdatedDual(const double label,
                                          const double example_weight,
                                          const double current_dual,
                                          const double wx,
                                          const double weighted_example_norm,
                                          const double primal_loss,
                                          const double dual_loss) {
    const double partial_derivative_loss =
        PartialDerivativeLogisticLoss(label, wx);
    // f(a) = sup (a*x  - f(x)) then a = f'(x), where a is the aproximate dual.
    const double approximate_dual = partial_derivative_loss * label;
    // Dual loss is gamma-strongly convex.
    const double gamma =
        1 / SmoothnessConstantLogisticLoss(partial_derivative_loss, label, wx);
    const double delta_dual = approximate_dual - current_dual;
    const double wx_dual = wx * current_dual * example_weight;
    const double delta_dual_squared = delta_dual * delta_dual;
    const double smooth_delta_dual_squared = delta_dual_squared * gamma * 0.5;
    double multiplier =
        (primal_loss + dual_loss + wx_dual + smooth_delta_dual_squared) /
        std::max(1.0,
                 delta_dual_squared *
                     (gamma +
                      weighted_example_norm * example_weight * example_weight));
    // Multiplier must be in the range [0, 1].
    multiplier = std::max(std::min(1.0, multiplier), 0.0);
    return current_dual + delta_dual * multiplier;
  }

  // Dual of logisitic loss function.
  // https://en.wikipedia.org/wiki/Convex_conjugate
  inline static double ComputeDualLoss(const double current_dual,
                                       const double example_label,
                                       const double example_weight) {
    // Dual of the logistic loss function is
    // ay * log(ay) + (1-ay) * log (1-ay), where a is the dual variable.
{'fraction_non_alphanumeric': 0.09370078740157481, 'fraction_numerical': 0.013910761154855643, 'mean_word_length': 2.381100266193434, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6985445', 'n_tokens_mistral': 2235, 'n_tokens_neox': 2021, 'n_words': 694}