<commit_before>/* Copyright 2016-present The KotlinNLP Authors. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 * ------------------------------------------------------------------*/

package com.kotlinnlp.simplednn.core.functionalities.losses

import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArray

/**
 * Softmax cross-entropy Calculator.
 *
 * It must be used to calculate the loss ONLY if the activation function of the
 * output is the [com.kotlinnlp.simplednn.core.functionalities.activations.Softmax].
 */
open class SoftmaxCrossEntropyCalculator : LossCalculator {

  companion object {

    /**
     * A values threshold to avoid underflow errors.
     */
    private const val EPS: Double = 1.0e-08
  }

  /**
   * Calculate the loss between an output and its gold.
   *
   *   -G * log(O)
   *
   * @param output the output prediction
   * @param outputGold the expected output
   *
   * @return the loss within [output] and [outputGold]
   */
  override fun calculateLoss(output: DenseNDArray, outputGold: DenseNDArray): DenseNDArray {

    require(outputGold.isOneHotEncoder) {
      "The gold output must be a one hot encoder to calculate the loss with the cross-entropy function."
    }

    val oneIndex: Int = outputGold.argMaxIndex()
    val loss: DenseNDArray = output.zerosLike()
    val argmaxOutput: Double = output[oneIndex]

    loss[oneIndex] = -Math.log(if (argmaxOutput >= EPS) argmaxOutput else EPS)

    return loss
  }

  /**
   * Calculate the errors between an output and its gold.
   *
   * @param output the output prediction
   * @param outputGold the expected output
   *
   * @return the derivative of the loss within [output] and [outputGold]
   */
  override fun calculateErrors(output: DenseNDArray, outputGold: DenseNDArray): DenseNDArray =
    output.sub(outputGold)

  /**
   * Calculate the errors between an output and its gold, given the one hot gold index.
   *
   * @param output the output prediction
   * @param goldIndex the index of the gold value
   *
   * @return the derivative of the loss within the [output] and its gold
   */
  fun calculateErrors(output: DenseNDArray, goldIndex: Int): DenseNDArray {

    val errors: DenseNDArray = output.copy()

    errors[goldIndex] = errors[goldIndex] - 1.0

    return errors
  }
}
<commit_msg>Add calculateErrors() variant<commit_after>/* Copyright 2016-present The KotlinNLP Authors. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 * ------------------------------------------------------------------*/

package com.kotlinnlp.simplednn.core.functionalities.losses

import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArray

/**
 * Softmax cross-entropy Calculator.
 *
 * It must be used to calculate the loss ONLY if the activation function of the
 * output is the [com.kotlinnlp.simplednn.core.functionalities.activations.Softmax].
 */
open class SoftmaxCrossEntropyCalculator : LossCalculator {

  companion object {

    /**
     * A values threshold to avoid underflow errors.
     */
    private const val EPS: Double = 1.0e-08
  }

  /**
   * Calculate the loss between an output and its gold.
   *
   *   -G * log(O)
   *
   * @param output the output prediction
   * @param outputGold the expected output
   *
   * @return the loss within [output] and [outputGold]
   */
  override fun calculateLoss(output: DenseNDArray, outputGold: DenseNDArray): DenseNDArray {

    require(outputGold.isOneHotEncoder) {
      "The gold output must be a one hot encoder to calculate the loss with the cross-entropy function."
    }

    val oneIndex: Int = outputGold.argMaxIndex()
    val loss: DenseNDArray = output.zerosLike()
    val argmaxOutput: Double = output[oneIndex]

    loss[oneIndex] = -Math.log(if (argmaxOutput >= EPS) argmaxOutput else EPS)

    return loss
  }

  /**
   * Calculate the errors between an output and its gold.
   *
   * @param output the output prediction
   * @param outputGold the expected output
   *
   * @return the derivative of the loss within [output] and [outputGold]
   */
  override fun calculateErrors(output: DenseNDArray, outputGold: DenseNDArray): DenseNDArray =
    output.sub(outputGold)

  /**
   * Calculate the errors between an output and its gold, given the one hot gold index.
   *
   * @param output the output prediction
   * @param goldIndex the index of the gold value
   *
   * @return the derivative of the loss within the [output] and its gold
   */
  fun calculateErrors(output: DenseNDArray, goldIndex: Int): DenseNDArray {

    val errors: DenseNDArray = output.copy()

    errors[goldIndex] = errors[goldIndex] - 1.0

    return errors
  }

  /**
   * Calculate the errors of a sequence.
   *
   * TODO: the [outputGoldSequence] should be a List<Int>
   *
   * @param outputSequence a list containing the output of the network for each example of a sequence
   * @param outputGoldSequence a list containing the indexes of the gold sequence
   *
   * @return an array containing the errors for each example of the sequence
   */
  fun calculateErrors(outputSequence: List<DenseNDArray>,
                      outputGoldSequence: ArrayList<Int>): List<DenseNDArray> {

    require(outputSequence.size == outputGoldSequence.size)

    return outputSequence.zip(outputGoldSequence).map {
      this.calculateErrors(output = it.first, goldIndex = it.second)
    }
  }
}

{'fraction_non_alphanumeric': 0.1068114165645246, 'fraction_numerical': 0.004902819120994572, 'mean_word_length': 2.98326359832636, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '539608', 'n_tokens_mistral': 1699, 'n_tokens_neox': 1612, 'n_words': 576}