<commit_before>/*
 *
 *  * Copyright 2016 Skymind,Inc.
 *  *
 *  *    Licensed under the Apache License, Version 2.0 (the "License");
 *  *    you may not use this file except in compliance with the License.
 *  *    You may obtain a copy of the License at
 *  *
 *  *        http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *    Unless required by applicable law or agreed to in writing, software
 *  *    distributed under the License is distributed on an "AS IS" BASIS,
 *  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *    See the License for the specific language governing permissions and
 *  *    limitations under the License.
 *
 */

package org.deeplearning4j.nn.graph.vertex.impl;

import org.deeplearning4j.berkeley.Pair;
import org.deeplearning4j.nn.api.Layer;
import org.deeplearning4j.nn.gradient.Gradient;
import org.deeplearning4j.nn.graph.ComputationGraph;
import org.deeplearning4j.nn.graph.vertex.BaseGraphVertex;
import org.deeplearning4j.nn.graph.vertex.VertexIndices;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastDivOp;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastMulOp;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastSubOp;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.ops.transforms.Transforms;

/**
 * L2NormalizeVertex performs L2 normalization on a single input.
 *
 * @author <NAME> (crockpotveggies)
 * @author <NAME> (AlexDBlack)
 */
public class L2NormalizeVertex extends BaseGraphVertex {

    private int[] dimension;
    private double eps;

    public L2NormalizeVertex(ComputationGraph graph, String name, int vertexIndex, int[] dimension, double eps){
        this(graph,name,vertexIndex,null,null,dimension,eps);
    }

    public L2NormalizeVertex(ComputationGraph graph, String name, int vertexIndex, VertexIndices[] inputVertices,
                             VertexIndices[] outputVertices, int[] dimension, double eps) {
        super(graph, name, vertexIndex, inputVertices, outputVertices);
        this.dimension = dimension;
        this.eps = eps;
    }

    @Override
    public boolean hasLayer() {
        return false;
    }

    @Override
    public boolean isOutputVertex() {
        return false;
    }

    @Override
    public Layer getLayer() {
        return null;
    }

    @Override
    public INDArray doForward(boolean training) {
        if(!canDoForward()) throw new IllegalStateException("Cannot do forward pass: inputs not set (L2NormalizeVertex "+vertexName+" idx "+vertexIndex+")");

        // L2 norm along all dimensions except 0, unless user-specified
        // x / |x|2
        INDArray x = inputs[0];
        int[] dimensions;
        if(dimension.length<1) {
            dimensions = new int[x.rank()-1];
            for( int i=1; i<x.rank(); i++ ){
                dimensions[i-1] = i;
            }
        } else {
            dimensions = dimension;
        }

        INDArray xNorm2 = x.norm2(dimensions)
        Transforms.max(xNorm2, eps, false); // in case of div by 0
        x.diviColumnVector(xNorm2);

        return x;

    }

    @Override
    public Pair<Gradient, INDArray[]> doBackward(boolean tbptt) {
        if(!canDoBackward()) throw new IllegalStateException("Cannot do backward pass: errors not set (L2NormalizeVertex "+vertexName+" idx "+vertexIndex+")");

        INDArray x = inputs[0];
        int[] dimensions;<commit_msg>Forgot ;<commit_after>/*
 *
 *  * Copyright 2016 Skymind,Inc.
 *  *
 *  *    Licensed under the Apache License, Version 2.0 (the "License");
 *  *    you may not use this file except in compliance with the License.
 *  *    You may obtain a copy of the License at
 *  *
 *  *        http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *    Unless required by applicable law or agreed to in writing, software
 *  *    distributed under the License is distributed on an "AS IS" BASIS,
 *  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *    See the License for the specific language governing permissions and
 *  *    limitations under the License.
 *
 */

package org.deeplearning4j.nn.graph.vertex.impl;

import org.deeplearning4j.berkeley.Pair;
import org.deeplearning4j.nn.api.Layer;
import org.deeplearning4j.nn.gradient.Gradient;
import org.deeplearning4j.nn.graph.ComputationGraph;
import org.deeplearning4j.nn.graph.vertex.BaseGraphVertex;
import org.deeplearning4j.nn.graph.vertex.VertexIndices;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastDivOp;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastMulOp;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastSubOp;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.ops.transforms.Transforms;

/**
 * L2NormalizeVertex performs L2 normalization on a single input.
 *
 * @author <NAME> (crockpotveggies)
 * @author <NAME> (AlexDBlack)
 */
public class L2NormalizeVertex extends BaseGraphVertex {

    private int[] dimension;
    private double eps;

    public L2NormalizeVertex(ComputationGraph graph, String name, int vertexIndex, int[] dimension, double eps){
        this(graph,name,vertexIndex,null,null,dimension,eps);
    }

    public L2NormalizeVertex(ComputationGraph graph, String name, int vertexIndex, VertexIndices[] inputVertices,
                             VertexIndices[] outputVertices, int[] dimension, double eps) {
        super(graph, name, vertexIndex, inputVertices, outputVertices);
        this.dimension = dimension;
        this.eps = eps;
    }

    @Override
    public boolean hasLayer() {
        return false;
    }

    @Override
    public boolean isOutputVertex() {
        return false;
    }

    @Override
    public Layer getLayer() {
        return null;
    }

    @Override
    public INDArray doForward(boolean training) {
        if(!canDoForward()) throw new IllegalStateException("Cannot do forward pass: inputs not set (L2NormalizeVertex "+vertexName+" idx "+vertexIndex+")");

        // L2 norm along all dimensions except 0, unless user-specified
        // x / |x|2
        INDArray x = inputs[0];
        int[] dimensions;
        if(dimension.length<1) {
            dimensions = new int[x.rank()-1];
            for( int i=1; i<x.rank(); i++ ){
                dimensions[i-1] = i;
            }
        } else {
            dimensions = dimension;
        }

        INDArray xNorm2 = x.norm2(dimensions);
        Transforms.max(xNorm2, eps, false); // in case of div by 0
        x.diviColumnVector(xNorm2);

        return x;

    }

    @Override
    public Pair<Gradient, INDArray[]> doBackward(boolean tbptt) {
        if(!canDoBackward()) throw new IllegalStateException("Cannot do backward pass: errors not set (L2NormalizeVertex "+vertexName+" idx "+vertexIndex+")");

        INDArray x = inputs[0];
        int[] dimensions;
{'fraction_non_alphanumeric': 0.10497397339502602, 'fraction_numerical': 0.012434933487565067, 'mean_word_length': 2.961626575028637, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4828627', 'n_tokens_mistral': 2138, 'n_tokens_neox': 2075, 'n_words': 552}