<commit_before>/*
 *
 *  * Copyright 2015 Skymind,Inc.
 *  *
 *  *    Licensed under the Apache License, Version 2.0 (the "License");
 *  *    you may not use this file except in compliance with the License.
 *  *    You may obtain a copy of the License at
 *  *
 *  *        http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *    Unless required by applicable law or agreed to in writing, software
 *  *    distributed under the License is distributed on an "AS IS" BASIS,
 *  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *    See the License for the specific language governing permissions and
 *  *    limitations under the License.
 *
 */

package org.deeplearning4j.nn.layers;

import java.io.*;

import org.deeplearning4j.berkeley.Pair;
import org.deeplearning4j.berkeley.Triple;
import org.deeplearning4j.eval.Evaluation;
import org.deeplearning4j.nn.api.Classifier;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.gradient.DefaultGradient;
import org.deeplearning4j.nn.gradient.Gradient;
import org.deeplearning4j.nn.params.DefaultParamInitializer;
import org.deeplearning4j.optimize.Solver;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.LossFunction;
import org.nd4j.linalg.api.ops.impl.transforms.SoftMax;
import org.nd4j.linalg.dataset.api.DataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.indexing.NDArrayIndex;
import org.nd4j.linalg.lossfunctions.LossCalculation;
import org.nd4j.linalg.lossfunctions.LossFunctions;
import org.nd4j.linalg.util.FeatureUtil;
import org.nd4j.linalg.util.LinAlgExceptions;

import static org.nd4j.linalg.ops.transforms.Transforms.pow;
import static org.nd4j.linalg.ops.transforms.Transforms.sqrt;


/**
 * Output layer with different objective
 * incooccurrences for different objectives.
 * This includes classification as well as prediction
 * @author <NAME>
 *
 */
public abstract class BaseOutputLayer<LayerConfT extends org.deeplearning4j.nn.conf.layers.BaseOutputLayer>
	extends BaseLayer<LayerConfT> implements Serializable,Classifier {

    //current input and label matrices
    protected INDArray labels;
    
    private transient Solver solver;
    
    private double fullNetworkL1;
    private double fullNetworkL2;

    public BaseOutputLayer(NeuralNetConfiguration conf) {
        super(conf);
    }

    public BaseOutputLayer(NeuralNetConfiguration conf, INDArray input) {
        super(conf, input);
    }
    
    /** Compute score after labels and input have been set.
     */
    public double computeScore( double fullNetworkL1, double fullNetworkL2 ){
    	if( input == null || labels == null )
    		throw new IllegalStateException("Cannot calculate score without input and labels");
    	this.fullNetworkL1 = fullNetworkL1;
    	this.fullNetworkL2 = fullNetworkL2;
    	setScoreWithZ(output2d(input));
    	return score;
    }

    @Override
    public void computeGradientAndScore() {
        if(input == null || labels == null)
            return;

        INDArray z = preOutput2d(input,true);
        Triple<Gradient,INDArray,INDArray> triple = getGradientsAndDelta(z);
        this.gradient = triple.getFirst();
        setScoreWithZ(triple.getThird());
    }

    @Override
    protected void setScoreWithZ(INDArray z) {
        if (layerConf().getLossFunction() == LossFunctions.LossFunction.CUSTOM) {
            LossFunction create = Nd4j.getOpFactory().createLossFunction(layerConf().getCustomLossFunction(), input, z);
            create.exec();
            score = create.currentResult().doubleValue();
        }

        else {
            score = LossCalculation.builder()
                    .l1(fullNetworkL1).l2(fullNetworkL2)
                    .labels(getLabels2d()).z(z).lossFunction(layerConf().getLossFunction())
                    .miniBatch(conf.isMiniBatch()).miniBatchSize(getInputMiniBatchSize())
                    .useRegularization(conf.isUseRegularization()).build().score();
        }
    }

    @Override
    public Pair<Gradient, Double> gradientAndScore() {
        return new Pair<>(gradient(),score());
    }

    @Override
    public Pair<Gradient,INDArray> backpropGradient(INDArray epsilon) {
    	Triple<Gradient,INDArray,INDArray> triple = getGradientsAndDelta(preOutput2d(input,true));	//Returns Gradient and delta^(this), not Gradient and epsilon^(this-1)
    	INDArray delta = triple.getSecond();

        INDArray epsilonNext = params.get(DefaultParamInitializer.WEIGHT_KEY).mmul(delta.transpose()).transpose();
        return new Pair<>(triple.getFirst(),epsilonNext);
    }

    /**
     * Gets the gradient from one training iteration
     * @return the gradient (bias and weight matrix)
     */
    @Override
    public Gradient gradient() {
        LinAlgExceptions.assertRows(input, getLabels2d());
        return gradient;

    }
    
    /** Returns tuple: {Gradient,Delta,Output} given preOut */
    private Triple<Gradient,INDArray,INDArray> getGradientsAndDelta(INDArray preOut){
    	INDArray output = Nd4j.getExecutioner().execAndReturn(Nd4j.getOpFactory().createTransform(conf().getLayer().getActivationFunction(), preOut.dup()));
    	INDArray outSubLabels = output.sub(getLabels2d());
    	Gradient gradient = new DefaultGradient();
    	
    	switch (layerConf().getLossFunction()) {
    	case MCXENT:	//cross-entropy (multi-class, with one-hot encoding)
    		gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(outSubLabels));
    		gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, outSubLabels.sum(0));
    		return new Triple<>(gradient,outSubLabels,output);
        case XENT: // cross-entropy (single binary output variable)
        	gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(outSubLabels.div(output.mul(output.rsub(1)))));
        	gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, outSubLabels.sum(0));
        	return new Triple<>(gradient,outSubLabels,output);

        case MSE: // mean squared error
        	INDArray delta = outSubLabels.mul(derivativeActivation(preOut));
        	gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(delta));
        	gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, delta.sum(0));
            return new Triple<>(gradient,delta,output);
        	
        case EXPLL: // exponential logarithmic
        	gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(labels.rsub(1).divi(output)));
        	gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, outSubLabels.sum(0));
            return new Triple<>(gradient,outSubLabels,output);
            
        case RMSE_XENT: // root mean squared error cross entropy
        	INDArray squaredrmseXentDiff = pow(outSubLabels, 2.0);
        	INDArray sqrt = sqrt(squaredrmseXentDiff);
        	gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(sqrt));
        	gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, outSubLabels.sum(0));
            return new Triple<>(gradient,outSubLabels,output);
        	
        case SQUARED_LOSS:
        	gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(input.transpose().mmul(pow(outSubLabels,2))));
        	gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, outSubLabels.sum(0));
            return new Triple<>(gradient,outSubLabels,output);
            
        case NEGATIVELOGLIKELIHOOD: // multi-class cross-entropy
        	gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(outSubLabels));
        	gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, outSubLabels.sum(0));
            return new Triple<>(gradient,outSubLabels,output);
        default:
        	throw new IllegalStateException("Invalid loss function: " + layerConf().getLossFunction());
    	}
    }


    @Override
    public INDArray activate(INDArray input, boolean training) {
        setInput(input, training);
        return output(training);
    }

    @Override
    public INDArray activate(INDArray input) {<commit_msg>fix squared loss calculation<commit_after>/*
 *
 *  * Copyright 2015 Skymind,Inc.
 *  *
 *  *    Licensed under the Apache License, Version 2.0 (the "License");
 *  *    you may not use this file except in compliance with the License.
 *  *    You may obtain a copy of the License at
 *  *
 *  *        http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *    Unless required by applicable law or agreed to in writing, software
 *  *    distributed under the License is distributed on an "AS IS" BASIS,
 *  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *    See the License for the specific language governing permissions and
 *  *    limitations under the License.
 *
 */

package org.deeplearning4j.nn.layers;

import java.io.*;

import org.deeplearning4j.berkeley.Pair;
import org.deeplearning4j.berkeley.Triple;
import org.deeplearning4j.eval.Evaluation;
import org.deeplearning4j.nn.api.Classifier;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.gradient.DefaultGradient;
import org.deeplearning4j.nn.gradient.Gradient;
import org.deeplearning4j.nn.params.DefaultParamInitializer;
import org.deeplearning4j.optimize.Solver;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.LossFunction;
import org.nd4j.linalg.api.ops.impl.transforms.SoftMax;
import org.nd4j.linalg.dataset.api.DataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.indexing.NDArrayIndex;
import org.nd4j.linalg.lossfunctions.LossCalculation;
import org.nd4j.linalg.lossfunctions.LossFunctions;
import org.nd4j.linalg.util.FeatureUtil;
import org.nd4j.linalg.util.LinAlgExceptions;

import static org.nd4j.linalg.ops.transforms.Transforms.pow;
import static org.nd4j.linalg.ops.transforms.Transforms.sqrt;


/**
 * Output layer with different objective
 * incooccurrences for different objectives.
 * This includes classification as well as prediction
 * @author <NAME>
 *
 */
public abstract class BaseOutputLayer<LayerConfT extends org.deeplearning4j.nn.conf.layers.BaseOutputLayer>
	extends BaseLayer<LayerConfT> implements Serializable,Classifier {

    //current input and label matrices
    protected INDArray labels;
    
    private transient Solver solver;
    
    private double fullNetworkL1;
    private double fullNetworkL2;

    public BaseOutputLayer(NeuralNetConfiguration conf) {
        super(conf);
    }

    public BaseOutputLayer(NeuralNetConfiguration conf, INDArray input) {
        super(conf, input);
    }
    
    /** Compute score after labels and input have been set.
     */
    public double computeScore( double fullNetworkL1, double fullNetworkL2 ){
    	if( input == null || labels == null )
    		throw new IllegalStateException("Cannot calculate score without input and labels");
    	this.fullNetworkL1 = fullNetworkL1;
    	this.fullNetworkL2 = fullNetworkL2;
    	setScoreWithZ(output2d(input));
    	return score;
    }

    @Override
    public void computeGradientAndScore() {
        if(input == null || labels == null)
            return;

        INDArray z = preOutput2d(input,true);
        Triple<Gradient,INDArray,INDArray> triple = getGradientsAndDelta(z);
        this.gradient = triple.getFirst();
        setScoreWithZ(triple.getThird());
    }

    @Override
    protected void setScoreWithZ(INDArray z) {
        if (layerConf().getLossFunction() == LossFunctions.LossFunction.CUSTOM) {
            LossFunction create = Nd4j.getOpFactory().createLossFunction(layerConf().getCustomLossFunction(), input, z);
            create.exec();
            score = create.currentResult().doubleValue();
        }

        else {
            score = LossCalculation.builder()
                    .l1(fullNetworkL1).l2(fullNetworkL2)
                    .labels(getLabels2d()).z(z).lossFunction(layerConf().getLossFunction())
                    .miniBatch(conf.isMiniBatch()).miniBatchSize(getInputMiniBatchSize())
                    .useRegularization(conf.isUseRegularization()).build().score();
        }
    }

    @Override
    public Pair<Gradient, Double> gradientAndScore() {
        return new Pair<>(gradient(),score());
    }

    @Override
    public Pair<Gradient,INDArray> backpropGradient(INDArray epsilon) {
    	Triple<Gradient,INDArray,INDArray> triple = getGradientsAndDelta(preOutput2d(input,true));	//Returns Gradient and delta^(this), not Gradient and epsilon^(this-1)
    	INDArray delta = triple.getSecond();

        INDArray epsilonNext = params.get(DefaultParamInitializer.WEIGHT_KEY).mmul(delta.transpose()).transpose();
        return new Pair<>(triple.getFirst(),epsilonNext);
    }

    /**
     * Gets the gradient from one training iteration
     * @return the gradient (bias and weight matrix)
     */
    @Override
    public Gradient gradient() {
        LinAlgExceptions.assertRows(input, getLabels2d());
        return gradient;

    }
    
    /** Returns tuple: {Gradient,Delta,Output} given preOut */
    private Triple<Gradient,INDArray,INDArray> getGradientsAndDelta(INDArray preOut){
    	INDArray output = Nd4j.getExecutioner().execAndReturn(Nd4j.getOpFactory().createTransform(conf().getLayer().getActivationFunction(), preOut.dup()));
    	INDArray outSubLabels = output.sub(getLabels2d());
    	Gradient gradient = new DefaultGradient();
    	
    	switch (layerConf().getLossFunction()) {
    	case MCXENT:	//cross-entropy (multi-class, with one-hot encoding)
    		gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(outSubLabels));
    		gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, outSubLabels.sum(0));
    		return new Triple<>(gradient,outSubLabels,output);
        case XENT: // cross-entropy (single binary output variable)
        	gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(outSubLabels.div(output.mul(output.rsub(1)))));
        	gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, outSubLabels.sum(0));
        	return new Triple<>(gradient,outSubLabels,output);

        case MSE: // mean squared error
        	INDArray delta = outSubLabels.mul(derivativeActivation(preOut));
        	gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(delta));
        	gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, delta.sum(0));
            return new Triple<>(gradient,delta,output);
        	
        case EXPLL: // exponential logarithmic
        	gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(labels.rsub(1).divi(output)));
        	gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, outSubLabels.sum(0));
            return new Triple<>(gradient,outSubLabels,output);
            
        case RMSE_XENT: // root mean squared error cross entropy
        	INDArray squaredrmseXentDiff = pow(outSubLabels, 2.0);
        	INDArray sqrt = sqrt(squaredrmseXentDiff);
        	gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(sqrt));
        	gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, outSubLabels.sum(0));
            return new Triple<>(gradient,outSubLabels,output);
        	
        case SQUARED_LOSS:
        	gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(pow(outSubLabels,2)));
        	gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, outSubLabels.sum(0));
            return new Triple<>(gradient,outSubLabels,output);
            
        case NEGATIVELOGLIKELIHOOD: // multi-class cross-entropy
        	gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, input.transpose().mmul(outSubLabels));
        	gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, outSubLabels.sum(0));
            return new Triple<>(gradient,outSubLabels,output);
        default:
        	throw new IllegalStateException("Invalid loss function: " + layerConf().getLossFunction());
    	}
    }


    @Override
    public INDArray activate(INDArray input, boolean training) {
        setInput(input, training);
        return output(training);
    }

    @Override
    public INDArray activate(INDArray input) {
{'fraction_non_alphanumeric': 0.10784722636000715, 'fraction_numerical': 0.007984269796818209, 'mean_word_length': 3.649307479224377, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3024477', 'n_tokens_mistral': 5090, 'n_tokens_neox': 4842, 'n_words': 929}