<commit_before>/* Copyright 2016-present The KotlinNLP Authors. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 * ------------------------------------------------------------------*/

package com.kotlinnlp.simplednn.deeplearning.pointernetwork

import com.kotlinnlp.simplednn.core.functionalities.activations.Tanh
import com.kotlinnlp.simplednn.core.layers.LayerType
import com.kotlinnlp.simplednn.deeplearning.attentionnetwork.attentionmechanism.AttentionMechanism
import com.kotlinnlp.simplednn.deeplearning.attentionnetwork.attentionmechanism.AttentionStructure
import com.kotlinnlp.simplednn.core.mergelayers.affine.AffineLayerStructure
import com.kotlinnlp.simplednn.core.mergelayers.affine.AffineLayersPool
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArray

/**
 * The forward helper of the [PointerNetwork].
 *
 * @property network the attentive recurrent network of this helper
 */
class ForwardHelper(private val network: PointerNetwork) {

  /**
   * The list of transform layers groups used during the last forward.
   */
  internal val usedTransformLayers = mutableListOf<List<AffineLayerStructure<DenseNDArray>>>()

  /**
   * The list of attention structures used during the last forward.
   */
  internal val usedAttentionStructures = mutableListOf<AttentionStructure>()

  /**
   * A pool of Affine Layers used to build the attention arrays.
   */
  private val transformLayersPool: AffineLayersPool<DenseNDArray> =
    AffineLayersPool(
      inputType = LayerType.Input.Dense,
      activationFunction = Tanh(),
      params = this.network.model.transformParams)

  /**
   * @param vector the vector that modulates a content-based attention mechanism over the input sequence
   *
   * @return an array that contains the importance score for each element of the input sequence
   */
  fun forward(vector: DenseNDArray): DenseNDArray {

    if (this.network.firstState) this.initForward()

    val attentionArrays: List<DenseNDArray> = this.buildAttentionSequence(vector)

    return AttentionMechanism(this.buildAttentionStructure(attentionArrays)).forward()
  }

  /**
   * @param attentionArrays the attention arrays
   *
   * @return the attention structure
   */
  private fun buildAttentionStructure(attentionArrays: List<DenseNDArray>): AttentionStructure {

    this.usedAttentionStructures.add(
      AttentionStructure(attentionArrays, params = this.network.model.attentionParams))

    return this.usedAttentionStructures.last()
  }

  /**
   * @param vector the vector that modulates a content-based attention mechanism over the input sequence
   *
   * @return the sequence of attention arrays
   */
  private fun buildAttentionSequence(vector: DenseNDArray): List<DenseNDArray> {

    val transformLayers = this.getTransformLayers(size = this.network.inputSequence.size)

    return ArrayList(transformLayers.zip(this.network.inputSequence).map { (layer, element) ->

      layer.setInput1(element)
      layer.setInput2(vector)
      layer.forward()

      layer.outputArray.values
    })
  }

  /**
   * Get an available group of transform layers, adding it into the usedTransformLayers list.
   *
   * @param size the number of transform layer to build
   *
   * @return an available transform layer
   */
  private fun getTransformLayers(size: Int): List<AffineLayerStructure<DenseNDArray>> {

    this.usedTransformLayers.add(
      List(size = size, init = { this.transformLayersPool.getItem() })
    )

    return this.usedTransformLayers.last()
  }

  /**
   * Initialize the structures used during the forward.
   */
  private fun initForward() {

    this.transformLayersPool.releaseAll()
    this.usedTransformLayers.clear()
    this.usedAttentionStructures.clear()
  }
}<commit_msg>Replace the affine structure with a generic merge layer structure; move the support structures to the network<commit_after>/* Copyright 2016-present The KotlinNLP Authors. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 * ------------------------------------------------------------------*/

package com.kotlinnlp.simplednn.deeplearning.pointernetwork

import com.kotlinnlp.simplednn.core.mergelayers.MergeLayer
import com.kotlinnlp.simplednn.deeplearning.attentionnetwork.attentionmechanism.AttentionMechanism
import com.kotlinnlp.simplednn.deeplearning.attentionnetwork.attentionmechanism.AttentionStructure
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArray

/**
 * The forward helper of the [PointerNetwork].
 *
 * @property network the attentive recurrent network of this helper
 */
class ForwardHelper(
  private val network: PointerNetwork
) {

  /**
   * @param vector the vector that modulates a content-based attention mechanism over the input sequence
   *
   * @return an array that contains the importance score for each element of the input sequence
   */
  fun forward(vector: DenseNDArray): DenseNDArray {

    if (this.network.firstState) this.initForward()

    val attentionArrays: List<DenseNDArray> = this.buildAttentionSequence(vector)

    return AttentionMechanism(this.buildAttentionStructure(attentionArrays)).forward()
  }

  /**
   * @param attentionArrays the attention arrays
   *
   * @return the attention structure
   */
  private fun buildAttentionStructure(attentionArrays: List<DenseNDArray>): AttentionStructure {

    this.network.usedAttentionStructures.add(
      AttentionStructure(attentionArrays, params = this.network.model.attentionParams))

    return this.network.usedAttentionStructures.last()
  }

  /**
   * @param vector the vector that modulates a content-based attention mechanism over the input sequence
   *
   * @return the sequence of attention arrays
   */
  private fun buildAttentionSequence(vector: DenseNDArray): List<DenseNDArray> {

    val transformLayers = this.getTransformLayers(size = this.network.inputSequence.size)

    return ArrayList(transformLayers.zip(this.network.inputSequence).map { (layer, element) ->

      layer.setInput1(element)
      layer.setInput2(vector)
      layer.forward()

      layer.outputArray.values
    })
  }

  /**
   * Get an available group of transform layers, adding it into the usedTransformLayers list.
   *
   * @param size the number of transform layer to build
   *
   * @return an available transform layer
   */
  private fun getTransformLayers(size: Int): List<MergeLayer<DenseNDArray>> {

    this.network.usedTransformLayers.add(
      List(size = size, init = { this.network.transformLayersPool.getItem() })
    )

    return this.network.usedTransformLayers.last()
  }

  /**
   * Initialize the structures used during the forward.
   */
  private fun initForward() {

    this.network.transformLayersPool.releaseAll()
    this.network.usedTransformLayers.clear()
    this.network.usedAttentionStructures.clear()
  }
}
{'fraction_non_alphanumeric': 0.103424178895877, 'fraction_numerical': 0.002795248078266946, 'mean_word_length': 3.751660026560425, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3876809', 'n_tokens_mistral': 2074, 'n_tokens_neox': 1958, 'n_words': 522}