<commit_before>/* Copyright 2016-present The KotlinNLP Authors. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 * ------------------------------------------------------------------*/

package com.kotlinnlp.simplednn.deeplearning.recurrentattentivedecoder

import com.kotlinnlp.simplednn.core.functionalities.activations.Tanh
import com.kotlinnlp.simplednn.core.functionalities.updatemethods.UpdateMethod
import com.kotlinnlp.simplednn.core.layers.LayerType
import com.kotlinnlp.simplednn.core.layers.feedforward.FeedforwardLayerStructure
import com.kotlinnlp.simplednn.core.neuralprocessor.feedforward.FeedforwardNeuralProcessor
import com.kotlinnlp.simplednn.core.neuralprocessor.feedforward.FeedforwardNeuralProcessorsPool
import com.kotlinnlp.simplednn.core.neuralprocessor.recurrent.RecurrentNeuralProcessor
import com.kotlinnlp.simplednn.core.optimizer.ScheduledUpdater
import com.kotlinnlp.simplednn.deeplearning.attentionnetwork.AttentionNetwork
import com.kotlinnlp.simplednn.deeplearning.attentionnetwork.AttentionNetworksPool
import com.kotlinnlp.simplednn.deeplearning.attentionnetwork.FeedforwardLayersPool
import com.kotlinnlp.simplednn.simplemath.ndarray.Shape
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArray
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArrayFactory

/**
 * The [RecurrentAttentiveNetwork].
 *
 * It encodes an input sequence of N arrays into a parallel output sequence of N arrays, giving the focus to the i-th
 * array at the i-th encoding step thanks to a recurrent context network.
 *
 * At each step the current state is encoded combining the whole input sequence with the recurrent context (that is
 * the memory of this system). Then the encoded state is forwarded to the output through the output network together
 * with the recurrent context.
 *
 * The recurrent context is encoded using an RNN that receives the last encoded state and a context label vector as
 * input.
 *
 *
 * @property model the model of the network
 * @property updateMethod the update method helper (Learning Rate, ADAM, AdaGrad, ...)
 */
class RecurrentAttentiveNetwork(
  val model: RecurrentAttentiveNetworkModel,
  val updateMethod: UpdateMethod<*>
) : ScheduledUpdater {

  /**
   * The size of the processing sequence (set at the first forward state).
   */
  var sequenceSize: Int = 0
    private set

  /**
   * A pool of Feedforward Layers used to build the attention arrays of the Attention Network.
   */
  val transformLayersPool: FeedforwardLayersPool<DenseNDArray> =
    FeedforwardLayersPool(
      inputType = LayerType.Input.Dense,
      activationFunction = Tanh(),
      params = this.model.transformParams)

  /**
   * A pool of Attention Networks used to encode the current state.
   */
  val stateEncodersPool: AttentionNetworksPool<DenseNDArray> =
    AttentionNetworksPool(
      model = this.model.attentionParams,
      inputType = LayerType.Input.Dense)

  /**
   * The processor for the recurrent context network.
   */
  val recurrentContextProcessor: RecurrentNeuralProcessor<DenseNDArray> =
    RecurrentNeuralProcessor(this.model.recurrentContextNetwork)

  /**
   * The pool of Feedforward Neural Processors used to interpolate the output of the Attention Network together with the
   * recurrent context.
   */
  val outputNetworkPool: FeedforwardNeuralProcessorsPool<DenseNDArray> =
    FeedforwardNeuralProcessorsPool(this.model.outputNetwork)

  /**
   * The zeros array used as null state encoding.
   */
  val initialStateEncoding = DenseNDArrayFactory.zeros(Shape(this.model.recurrentContextSize))

  /**
   * The list of transform layers groups used during the last forward.
   */
  val usedTransformLayers = mutableListOf<List<FeedforwardLayerStructure<DenseNDArray>>>()

  /**<commit_msg>Update documentation<commit_after>/* Copyright 2016-present The KotlinNLP Authors. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 * ------------------------------------------------------------------*/

package com.kotlinnlp.simplednn.deeplearning.recurrentattentivedecoder

import com.kotlinnlp.simplednn.core.functionalities.activations.Tanh
import com.kotlinnlp.simplednn.core.functionalities.updatemethods.UpdateMethod
import com.kotlinnlp.simplednn.core.layers.LayerType
import com.kotlinnlp.simplednn.core.layers.feedforward.FeedforwardLayerStructure
import com.kotlinnlp.simplednn.core.neuralprocessor.feedforward.FeedforwardNeuralProcessor
import com.kotlinnlp.simplednn.core.neuralprocessor.feedforward.FeedforwardNeuralProcessorsPool
import com.kotlinnlp.simplednn.core.neuralprocessor.recurrent.RecurrentNeuralProcessor
import com.kotlinnlp.simplednn.core.optimizer.ScheduledUpdater
import com.kotlinnlp.simplednn.deeplearning.attentionnetwork.AttentionNetwork
import com.kotlinnlp.simplednn.deeplearning.attentionnetwork.AttentionNetworksPool
import com.kotlinnlp.simplednn.deeplearning.attentionnetwork.FeedforwardLayersPool
import com.kotlinnlp.simplednn.simplemath.ndarray.Shape
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArray
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArrayFactory

/**
 * The [RecurrentAttentiveNetwork].
 *
 * It encodes an input sequence of N arrays into a parallel output sequence of N arrays, giving the focus to the i-th
 * array at the i-th encoding step thanks to a recurrent context network.
 *
 * At each step the current state is encoded combining the whole input sequence with the recurrent context (that is
 * the memory of this system). Then the encoded state is forwarded to the output through the output network together
 * with the recurrent context.
 *
 * The recurrent context is encoded using an RNN that receives the last encoded state and a context label vector as
 * input.
 *
 *
 * @property model the model of the network
 * @property updateMethod the update method helper (Learning Rate, ADAM, AdaGrad, ...)
 */
class RecurrentAttentiveNetwork(
  val model: RecurrentAttentiveNetworkModel,
  val updateMethod: UpdateMethod<*>
) : ScheduledUpdater {

  /**
   * The size of the currently processing sequence (set at the first forward state).
   */
  var sequenceSize: Int = 0
    private set

  /**
   * A pool of Feedforward Layers used to build the attention arrays of the Attention Network.
   */
  val transformLayersPool: FeedforwardLayersPool<DenseNDArray> =
    FeedforwardLayersPool(
      inputType = LayerType.Input.Dense,
      activationFunction = Tanh(),
      params = this.model.transformParams)

  /**
   * A pool of Attention Networks used to encode the current state.
   */
  val stateEncodersPool: AttentionNetworksPool<DenseNDArray> =
    AttentionNetworksPool(
      model = this.model.attentionParams,
      inputType = LayerType.Input.Dense)

  /**
   * The processor of the recurrent context network.
   */
  val recurrentContextProcessor: RecurrentNeuralProcessor<DenseNDArray> =
    RecurrentNeuralProcessor(this.model.recurrentContextNetwork)

  /**
   * The pool of Feedforward Neural Processors used to interpolate the encoded state together with the recurrent
   * context.
   */
  val outputNetworkPool: FeedforwardNeuralProcessorsPool<DenseNDArray> =
    FeedforwardNeuralProcessorsPool(this.model.outputNetwork)

  /**
   * The zeros array used as encoding of the initial state.
   */
  val initialStateEncoding = DenseNDArrayFactory.zeros(Shape(this.model.recurrentContextSize))

  /**
   * The list of transform layers groups used during the last forward.
   */
  val usedTransformLayers = mutableListOf<List<FeedforwardLayerStructure<DenseNDArray>>>()

  /**
{'fraction_non_alphanumeric': 0.08795362903225806, 'fraction_numerical': 0.0022681451612903227, 'mean_word_length': 4.249338624338624, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1225662', 'n_tokens_mistral': 2339, 'n_tokens_neox': 2188, 'n_words': 641}