<commit_before>/* Copyright 2016-present The KotlinNLP Authors. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 * ------------------------------------------------------------------*/

package com.kotlinnlp.simplednn.core.optimizer

import com.kotlinnlp.simplednn.core.arrays.UpdatableDenseArray
import com.kotlinnlp.simplednn.core.functionalities.updatemethods.UpdateMethod
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArray
import com.kotlinnlp.simplednn.simplemath.ndarray.sparse.SparseNDArray

/**
 * The optimizer of neural parameters.
 *
 * @param params the parameters to optimize
 * @param updateMethod the update method helper (Learning Rate, ADAM, AdaGrad, ...)
 */
class ParamsOptimizer<ParamsType: IterableParams<ParamsType>>(
  val params: ParamsType,
  updateMethod: UpdateMethod<*>
) : Optimizer(updateMethod) {

  /**
   * The accumulator of parameters errors.
   */
  private val paramsErrorsAccumulator = ParamsErrorsAccumulator<ParamsType>()

  /**
   * A Boolean indicating if errors have been accumulated.
   */
  private var isEmpty: Boolean = false

  /**
   * A Boolean indicating if no errors have been accumulated.
   */
  private val isNotEmpty: Boolean get() = !this.isEmpty

  /**
   * Calculate the errors average, update the parameters.
   */
  override fun update() {

    if (this.isNotEmpty) {
      
      this.paramsErrorsAccumulator.averageErrors()
      this.updateParams()
      this.paramsErrorsAccumulator.reset()

      this.isEmpty = true
    }
  }

  /**
   * Accumulate the given [paramsErrors] into the accumulator.
   *
   * @param paramsErrors the parameters errors to accumulate
   * @param copy a Boolean indicating if the [paramsErrors] can be used as reference or must be copied. Set copy = false
   *             to optimize the accumulation when the amount of the errors to accumulate is 1. (default = true)
   */
  fun accumulate(paramsErrors: ParamsType, copy: Boolean = true) {

    require(this.params.zip(paramsErrors).all{ (params, errors) -> params.values.shape == errors.values.shape }) {
      "paramsErrors contains arrays with not compatible shape"
    }

    this.paramsErrorsAccumulator.accumulate(paramsErrors = paramsErrors, copy = copy)

    this.isEmpty = false
  }

  /**
   * Update the parameters in respect of the errors using the update method helper (Learning Rate, ADAM, AdaGrad, ...).
   */
  private fun updateParams() {

    this.params.zip(this.paramsErrorsAccumulator.getParamsErrors(copy = false)).forEach {
      (params, errors) ->

      val e = errors.values

      when (e) {
        is DenseNDArray -> this.updateMethod.update(array = params as UpdatableDenseArray, errors = e)
        is SparseNDArray -> this.updateMethod.update(array = params as UpdatableDenseArray, errors = e)
        else -> throw RuntimeException("Invalid errors type")
      }
    }
  }
}<commit_msg>Replace isEmpty and isNotEmpty with paramsErrorsAccumulator methods<commit_after>/* Copyright 2016-present The KotlinNLP Authors. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, you can obtain one at http://mozilla.org/MPL/2.0/.
 * ------------------------------------------------------------------*/

package com.kotlinnlp.simplednn.core.optimizer

import com.kotlinnlp.simplednn.core.arrays.UpdatableDenseArray
import com.kotlinnlp.simplednn.core.functionalities.updatemethods.UpdateMethod
import com.kotlinnlp.simplednn.simplemath.ndarray.dense.DenseNDArray
import com.kotlinnlp.simplednn.simplemath.ndarray.sparse.SparseNDArray

/**
 * The optimizer of neural parameters.
 *
 * @param params the parameters to optimize
 * @param updateMethod the update method helper (Learning Rate, ADAM, AdaGrad, ...)
 */
class ParamsOptimizer<ParamsType: IterableParams<ParamsType>>(
  val params: ParamsType,
  updateMethod: UpdateMethod<*>
) : Optimizer(updateMethod) {

  /**
   * The accumulator of parameters errors.
   */
  private val paramsErrorsAccumulator = ParamsErrorsAccumulator<ParamsType>()

  /**
   * Calculate the errors average, update the parameters.
   */
  override fun update() {

    if (this.paramsErrorsAccumulator.isNotEmpty) {
      
      this.paramsErrorsAccumulator.averageErrors()
      this.updateParams()
      this.paramsErrorsAccumulator.reset()
    }
  }

  /**
   * Accumulate the given [paramsErrors] into the accumulator.
   *
   * @param paramsErrors the parameters errors to accumulate
   * @param copy a Boolean indicating if the [paramsErrors] can be used as reference or must be copied. Set copy = false
   *             to optimize the accumulation when the amount of the errors to accumulate is 1. (default = true)
   */
  fun accumulate(paramsErrors: ParamsType, copy: Boolean = true) {

    require(this.params.zip(paramsErrors).all{ (params, errors) -> params.values.shape == errors.values.shape }) {
      "paramsErrors contains arrays with not compatible shape"
    }

    this.paramsErrorsAccumulator.accumulate(paramsErrors = paramsErrors, copy = copy)
  }

  /**
   * Update the parameters in respect of the errors using the update method helper (Learning Rate, ADAM, AdaGrad, ...).
   */
  private fun updateParams() {

    this.params.zip(this.paramsErrorsAccumulator.getParamsErrors(copy = false)).forEach {
      (params, errors) ->

      val e = errors.values

      when (e) {
        is DenseNDArray -> this.updateMethod.update(array = params as UpdatableDenseArray, errors = e)
        is SparseNDArray -> this.updateMethod.update(array = params as UpdatableDenseArray, errors = e)
        else -> throw RuntimeException("Invalid errors type")
      }
    }
  }
}
{'fraction_non_alphanumeric': 0.11566469093988145, 'fraction_numerical': 0.003048264182895851, 'mean_word_length': 3.173851590106007, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1920562', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1637, 'n_words': 496}