<commit_before>/* Copyright 2016-present The KotlinNLP Authors. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 * ------------------------------------------------------------------*/

package com.kotlinnlp.simplednn.core.layers

import com.kotlinnlp.simplednn.simplemath.ndarray.NDArray
import com.kotlinnlp.simplednn.simplemath.ndarray.NDArrayMask
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArray
import java.util.*

/**
 * The helper which executes the backward on a [layer].
 */
interface BackwardHelper<InputNDArrayType : NDArray<InputNDArrayType>> {

  /**
   * The [LayerStructure] in which the backward is executed.
   */
  val layer: LayerStructure<InputNDArrayType>

  /**
   * Executes the backward calculating the errors of the parameters and eventually of the input through the SGD
   * algorithm, starting from the preset errors of the output array.
   *
   * @param paramsErrors the errors of the parameters which will be filled
   * @param propagateToInput whether to propagate the errors to the input array
   * @param mePropK the k factor of the 'meProp' algorithm to propagate from the k (in percentage) output nodes with
   *                the top errors (ignored if null, the default)
   */
  fun backward(paramsErrors: LayerParameters<*>, propagateToInput: Boolean = false, mePropK: Double?)

  /**
   * @param mePropK the k factor of the 'meProp' algorithm to propagate from the k (in percentage) output nodes with
   *                the top errors
   *
   * @return the mask of the top k ([mePropK]) elements of the output array.
   */
  fun getOutputMask(mePropK: Double): NDArrayMask {

    val y: DenseNDArray = this.layer.outputArray.values
    val nTopElements: Int = Math.round(mePropK * y.length).toInt()

    val minHeap = PriorityQueue<Pair<Int, Double>>(y.length, Comparator({ a, b ->
      val diff: Double = a.second - b.second
      if (diff > 0.0) -1 else if (diff < 0.0) 1 else 0
    }))

    (0 until y.length).forEach { i -> minHeap.add(Pair(i, y[i])) }

    return NDArrayMask(
      dim1 = Array(size = nTopElements, init = { minHeap.remove().first }),
      dim2 = Array(size = nTopElements, init = { 0 }))
  }
}<commit_msg>Fix top k output errors selection<commit_after>/* Copyright 2016-present The KotlinNLP Authors. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 * ------------------------------------------------------------------*/

package com.kotlinnlp.simplednn.core.layers

import com.kotlinnlp.simplednn.simplemath.ndarray.NDArray
import com.kotlinnlp.simplednn.simplemath.ndarray.NDArrayMask
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArray
import java.util.*

/**
 * The helper which executes the backward on a [layer].
 */
interface BackwardHelper<InputNDArrayType : NDArray<InputNDArrayType>> {

  /**
   * The [LayerStructure] in which the backward is executed.
   */
  val layer: LayerStructure<InputNDArrayType>

  /**
   * Executes the backward calculating the errors of the parameters and eventually of the input through the SGD
   * algorithm, starting from the preset errors of the output array.
   *
   * @param paramsErrors the errors of the parameters which will be filled
   * @param propagateToInput whether to propagate the errors to the input array
   * @param mePropK the k factor of the 'meProp' algorithm to propagate from the k (in percentage) output nodes with
   *                the top errors (ignored if null, the default)
   */
  fun backward(paramsErrors: LayerParameters<*>, propagateToInput: Boolean = false, mePropK: Double?)

  /**
   * @param mePropK the k factor of the 'meProp' algorithm to propagate from the k (in percentage) output nodes with
   *                the top errors
   *
   * @return the mask of the k output nodes with the top errors
   */
  fun getOutputMask(mePropK: Double): NDArrayMask {

    val gy: DenseNDArray = this.layer.outputArray.errors
    val nTopElements: Int = Math.round(mePropK * gy.length).toInt()

    val minHeap = PriorityQueue<Pair<Int, Double>>(gy.length, Comparator({ a, b ->
      val diff: Double = a.second - b.second // compare errors values
      if (diff > 0.0) -1 else if (diff < 0.0) 1 else 0 // descending order
    }))

    (0 until gy.length).forEach { i -> minHeap.add(Pair(i, Math.abs(gy[i]))) } // pairs of <index, value>

    return NDArrayMask(
      dim1 = Array(size = nTopElements, init = { minHeap.remove().first }), // get top errors indices
      dim2 = Array(size = nTopElements, init = { 0 }))
  }
}
{'fraction_non_alphanumeric': 0.12252883031301483, 'fraction_numerical': 0.007825370675453048, 'mean_word_length': 2.9876847290640396, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3873528', 'n_tokens_mistral': 1471, 'n_tokens_neox': 1393, 'n_words': 515}