<commit_before><?php

namespace mcordingley\Regression\Algorithm\GradientDescent\Schedule;

/**
 * @package mcordingley\Regression\Algorithm\GradientDescent\Schedule
 * @link http://sebastianruder.com/optimizing-gradient-descent/index.html#adam
 */
final class Adam implements Schedule
{
    /** @var float */
    private $eta;

    /** @var array */
    private $gradient;

    /** @var int */
    private $iteration = 0;

    /** @var float */
    private $meanBeta;

    /** @var array */
    private $means;

    /** @var float */
    private $stepSize;

    /** @var float */
    private $varianceBeta;

    /** @var array */
    private $variances;

    /**
     * @param float $stepSize
     * @param float $eta
     * @param float $meanBeta
     * @param float $varianceBeta
     */
    public function __construct($stepSize = 0.001, $eta = 0.00000001, $meanBeta = 0.9, $varianceBeta = 0.999)
    {
        $this->stepSize = $stepSize;
        $this->eta = $eta;
        $this->meanBeta = $meanBeta;
        $this->varianceBeta = $varianceBeta;
    }

    /**
     * @param array $gradient
     */
    public function update(array $gradient)
    {
        if (!$this->means) {
            $this->means = array_fill(0, count($gradient), 0.0);
            $this->variances = array_fill(0, count($gradient), 0.0);
        }

        foreach ($gradient as $i => $slope) {
            $this->means[$i] = $this->meanBeta * $this->means[$i] + (1.0  - $this->meanBeta) * $slope;
            $this->variances[$i] = $this->varianceBeta * $this->variances[$i] + (1.0  - $this->varianceBeta) * pow($slope, 2);
        }

        $this->iteration++;
        $this->gradient = $gradient;
    }

    /**
     * @param int $featureIndex
     * @return float
     */
    public function step($featureIndex)
    {
        if (!$this->gradient[$featureIndex]) {
            return 0.0;
        }

        $correctedMean = $this->means[$featureIndex] / (1.0 - pow($this->meanBeta, $this->iteration));
        $correctedVariance = $this->variances[$featureIndex] / (1.0 - pow($this->varianceBeta, $this->iteration));

        /*
         * Need to put the gradient in the denominator here to counter the one in GradientDescent, since Adam takes
         * the unusual approach of not having it at all in the coefficient update step.
         */
        return $this->stepSize * $correctedMean / ((sqrt($correctedVariance) + $this->eta) * $this->gradient[$featureIndex]);
    }
}<commit_msg>Remove branch that will never execute.<commit_after><?php

namespace mcordingley\Regression\Algorithm\GradientDescent\Schedule;

/**
 * @package mcordingley\Regression\Algorithm\GradientDescent\Schedule
 * @link http://sebastianruder.com/optimizing-gradient-descent/index.html#adam
 */
final class Adam implements Schedule
{
    /** @var float */
    private $eta;

    /** @var array */
    private $gradient;

    /** @var int */
    private $iteration = 0;

    /** @var float */
    private $meanBeta;

    /** @var array */
    private $means;

    /** @var float */
    private $stepSize;

    /** @var float */
    private $varianceBeta;

    /** @var array */
    private $variances;

    /**
     * @param float $stepSize
     * @param float $eta
     * @param float $meanBeta
     * @param float $varianceBeta
     */
    public function __construct($stepSize = 0.001, $eta = 0.00000001, $meanBeta = 0.9, $varianceBeta = 0.999)
    {
        $this->stepSize = $stepSize;
        $this->eta = $eta;
        $this->meanBeta = $meanBeta;
        $this->varianceBeta = $varianceBeta;
    }

    /**
     * @param array $gradient
     */
    public function update(array $gradient)
    {
        if (!$this->means) {
            $this->means = array_fill(0, count($gradient), 0.0);
            $this->variances = array_fill(0, count($gradient), 0.0);
        }

        foreach ($gradient as $i => $slope) {
            $this->means[$i] = $this->meanBeta * $this->means[$i] + (1.0  - $this->meanBeta) * $slope;
            $this->variances[$i] = $this->varianceBeta * $this->variances[$i] + (1.0  - $this->varianceBeta) * pow($slope, 2);
        }

        $this->iteration++;
        $this->gradient = $gradient;
    }

    /**
     * @param int $featureIndex
     * @return float
     */
    public function step($featureIndex)
    {
        $correctedMean = $this->means[$featureIndex] / (1.0 - pow($this->meanBeta, $this->iteration));
        $correctedVariance = $this->variances[$featureIndex] / (1.0 - pow($this->varianceBeta, $this->iteration));

        /*
         * Need to put the gradient in the denominator here to counter the one in GradientDescent, since Adam takes
         * the unusual approach of not having it at all in the coefficient update step.
         */
        return $this->stepSize * $correctedMean / ((sqrt($correctedVariance) + $this->eta) * $this->gradient[$featureIndex]);
    }
}
{'fraction_non_alphanumeric': 0.15846436593832958, 'fraction_numerical': 0.014702879313865632, 'mean_word_length': 2.3686382393397523, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3513317', 'n_tokens_mistral': 1653, 'n_tokens_neox': 1529, 'n_words': 347}