<commit_before>/* Copyright 2016-present The KotlinNLP Authors. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 * ------------------------------------------------------------------*/

package com.kotlinnlp.simplednn.core.layers.recurrent.deltarnn

import com.kotlinnlp.simplednn.core.arrays.AugmentedArray
import com.kotlinnlp.simplednn.core.functionalities.activations.ActivationFunction
import com.kotlinnlp.simplednn.core.functionalities.activations.Sigmoid
import com.kotlinnlp.simplednn.core.functionalities.activations.Tanh
import com.kotlinnlp.simplednn.core.layers.*
import com.kotlinnlp.simplednn.core.layers.recurrent.*
import com.kotlinnlp.simplednn.simplemath.ndarray.NDArray
import com.kotlinnlp.simplednn.simplemath.ndarray.Shape
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArray
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArrayFactory

/**
 * The DeltaRNN Layer Structure.
 *
 * @property inputArray the input array of the layer
 * @property outputArray the output array of the layer
 * @property params the parameters which connect the input to the output
 * @property layerContextWindow the context window used for the forward and the backward
 * @property activationFunction the activation function of the layer
 * @property dropout the probability of dropout (default 0.0).
 *                   If applying it, the usual value is 0.5 (better 0.25 if it's the first layer).
 */
class DeltaRNNLayerStructure<InputNDArrayType : NDArray<InputNDArrayType>>(
  inputArray: AugmentedArray<InputNDArrayType>,
  outputArray: AugmentedArray<DenseNDArray>,
  params: LayerParameters,
  layerContextWindow: LayerContextWindow,
  activationFunction: ActivationFunction? = null,
  dropout: Double = 0.0
) : GatedRecurrentLayerStructure<InputNDArrayType>(
  inputArray = inputArray,
  outputArray = outputArray,
  params = params,
  layerContextWindow = layerContextWindow,
  activationFunction = activationFunction,
  dropout = dropout) {

  /**
   * The candidate array.
   */
  val candidate = AugmentedArray<DenseNDArray>(values = DenseNDArrayFactory.emptyArray(Shape(this.outputArray.size)))

  /**
   * The partition array.
   */
  val partition = AugmentedArray<DenseNDArray>(values = DenseNDArrayFactory.emptyArray(Shape(this.outputArray.size)))

  /**
   * The array which contains the result of the dot product between the input (x) and the input weights (w).
   */
  val wx = AugmentedArray<DenseNDArray>(values = DenseNDArrayFactory.emptyArray(Shape(this.outputArray.size)))

  /**
   * The array which contains the result of the dot product between the output in the previous state and the recurrent
   * weights.
   */
  val wyRec = AugmentedArray<DenseNDArray>(values = DenseNDArrayFactory.emptyArray(Shape(this.outputArray.size)))

  /**
   * The helper which execute the forward
   */
  override val forwardHelper = DeltaRNNForwardHelper(layer = this)

  /**
   * The helper which execute the backward
   */
  override val backwardHelper = DeltaRNNBackwardHelper(layer = this)

  /**
   * The helper which calculates the relevance
   */
  override val relevanceHelper = DeltaRNNRelevanceHelper(layer = this)

  /**
   * Initialization: set the activation functions.
   */
  init {

    if (activationFunction != null) {
      outputArray.setActivation(activationFunction)
    }

    this.candidate.setActivation(activationFunction ?: Tanh())
    this.partition.setActivation(Sigmoid())
  }
}
<commit_msg>Add relevanceSupport attribute<commit_after>/* Copyright 2016-present The KotlinNLP Authors. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 * ------------------------------------------------------------------*/

package com.kotlinnlp.simplednn.core.layers.recurrent.deltarnn

import com.kotlinnlp.simplednn.core.arrays.AugmentedArray
import com.kotlinnlp.simplednn.core.functionalities.activations.ActivationFunction
import com.kotlinnlp.simplednn.core.functionalities.activations.Sigmoid
import com.kotlinnlp.simplednn.core.functionalities.activations.Tanh
import com.kotlinnlp.simplednn.core.layers.*
import com.kotlinnlp.simplednn.core.layers.recurrent.*
import com.kotlinnlp.simplednn.simplemath.ndarray.NDArray
import com.kotlinnlp.simplednn.simplemath.ndarray.Shape
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArray
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArrayFactory

/**
 * The DeltaRNN Layer Structure.
 *
 * @property inputArray the input array of the layer
 * @property outputArray the output array of the layer
 * @property params the parameters which connect the input to the output
 * @property layerContextWindow the context window used for the forward and the backward
 * @property activationFunction the activation function of the layer
 * @property dropout the probability of dropout (default 0.0).
 *                   If applying it, the usual value is 0.5 (better 0.25 if it's the first layer).
 */
class DeltaRNNLayerStructure<InputNDArrayType : NDArray<InputNDArrayType>>(
  inputArray: AugmentedArray<InputNDArrayType>,
  outputArray: AugmentedArray<DenseNDArray>,
  params: LayerParameters,
  layerContextWindow: LayerContextWindow,
  activationFunction: ActivationFunction? = null,
  dropout: Double = 0.0
) : GatedRecurrentLayerStructure<InputNDArrayType>(
  inputArray = inputArray,
  outputArray = outputArray,
  params = params,
  layerContextWindow = layerContextWindow,
  activationFunction = activationFunction,
  dropout = dropout) {

  /**
   * The candidate array.
   */
  val candidate = AugmentedArray<DenseNDArray>(values = DenseNDArrayFactory.emptyArray(Shape(this.outputArray.size)))

  /**
   * The partition array.
   */
  val partition = AugmentedArray<DenseNDArray>(values = DenseNDArrayFactory.emptyArray(Shape(this.outputArray.size)))

  /**
   * The array which contains the result of the dot product between the input (x) and the input weights (w).
   */
  val wx = AugmentedArray<DenseNDArray>(values = DenseNDArrayFactory.emptyArray(Shape(this.outputArray.size)))

  /**
   * The array which contains the result of the dot product between the output in the previous state and the recurrent
   * weights.
   */
  val wyRec = AugmentedArray<DenseNDArray>(values = DenseNDArrayFactory.emptyArray(Shape(this.outputArray.size)))

  /**
   * The support structure used to save temporary results during a forward and using them to calculate the relevance
   * later.
   */
  val relevanceSupport: DeltaRNNRelevanceSupport
    get() = try {
      this._relevanceSupport

    } catch (e: UninitializedPropertyAccessException) {
      this._relevanceSupport = DeltaRNNRelevanceSupport(outputSize = this.outputArray.size)
      this._relevanceSupport
    }

  /**
   * The helper which execute the forward
   */
  override val forwardHelper = DeltaRNNForwardHelper(layer = this)

  /**
   * The helper which execute the backward
   */
  override val backwardHelper = DeltaRNNBackwardHelper(layer = this)

  /**
   * The helper which calculates the relevance
   */
  override val relevanceHelper = DeltaRNNRelevanceHelper(layer = this)

  /**
   * The support structure used to save temporary results during a forward and using them to calculate the relevance
   * later.
   */
  lateinit private var _relevanceSupport: DeltaRNNRelevanceSupport

  /**
   * Initialization: set the activation functions.
   */
  init {

    if (activationFunction != null) {
      outputArray.setActivation(activationFunction)
    }

    this.candidate.setActivation(activationFunction ?: Tanh())
    this.partition.setActivation(Sigmoid())
  }
}

{'fraction_non_alphanumeric': 0.101209982354424, 'fraction_numerical': 0.004285354171918326, 'mean_word_length': 4.009469696969697, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '144940', 'n_tokens_mistral': 2352, 'n_tokens_neox': 2206, 'n_words': 612}