<commit_before>package org.nd4j.linalg.dataset.api.preprocessor;

import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastAddOp;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastDivOp;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastMulOp;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastSubOp;
import org.nd4j.linalg.dataset.api.DataSet;
import org.nd4j.linalg.dataset.api.DataSetUtil;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.ops.transforms.Transforms;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;

/**
 * Created by susaneraly on 5/25/16.
 * Standard scaler calculates a moving column wise
 * variance and mean
 * http://www.johndcook.com/blog/standard_deviation/
 */
public class NormalizerStandardize implements DataNormalization {
    private static Logger logger = LoggerFactory.getLogger(NormalizerStandardize.class);
    private int runningTotal , labelRunningTotal = 0;
    private int batchCount,labelbatchCount = 0;
    private int featureRank = 2;
    private INDArray featureMeanStd, labelMeanStd;
    private INDArray featureMean, featureStd, labelMean, labelStd;
    private boolean fitLabels = false;

    private INDArray fit(INDArray theArray) {
        INDArray theMean, theStd;
        theMean = theArray.mean(0);
        theStd = theArray.std(0);
        theStd.addi(Nd4j.scalar(Nd4j.EPS_THRESHOLD));
        if (theStd.min(1) == Nd4j.scalar(Nd4j.EPS_THRESHOLD))
            logger.info("API_INFO: Std deviation found to be zero. Transform will round upto epsilon to avoid nans.");
        return Nd4j.vstack(theMean,theStd).dup();
    }

    private void runningFit(INDArray thenewArray, INDArray currentMeanStd, int runningTotal, boolean allDone) {
        int batchCount = thenewArray.size(0);
        INDArray currentMean = currentMeanStd.getRow(0);
        INDArray currentStd = currentMeanStd.getRow(1);
        if (!allDone) {
            // m_newM = m_oldM + (x - m_oldM)/m_n;
            INDArray xMinusMean = thenewArray.subRowVector(currentMean);
            INDArray newMean = currentMean.add(xMinusMean.sum(0).divi(runningTotal));
            // Using http://i.stanford.edu/pub/cstr/reports/cs/tr/79/773/CS-TR-79-773.pdf<commit_msg>NormalizerStandardize to take constructor arguments<commit_after>package org.nd4j.linalg.dataset.api.preprocessor;

import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastAddOp;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastDivOp;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastMulOp;
import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastSubOp;
import org.nd4j.linalg.dataset.api.DataSet;
import org.nd4j.linalg.dataset.api.DataSetUtil;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.ops.transforms.Transforms;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;

/**
 * Created by susaneraly on 5/25/16.
 * Standard scaler calculates a moving column wise
 * variance and mean
 * http://www.johndcook.com/blog/standard_deviation/
 */
public class NormalizerStandardize implements DataNormalization {
    private static Logger logger = LoggerFactory.getLogger(NormalizerStandardize.class);
    private int runningTotal , labelRunningTotal = 0;
    private int batchCount,labelbatchCount = 0;
    private int featureRank = 2;
    private INDArray featureMeanStd, labelMeanStd;
    private INDArray featureMean, featureStd, labelMean, labelStd;
    private boolean fitLabels = false;

    public NormalizerStandardize() {
    }

    public NormalizerStandardize(INDArray featureMean, INDArray featureStd) {
        this.featureMean = featureMean;
        this.featureStd = featureStd;
        this.fitLabels = false;
    }

    public NormalizerStandardize(INDArray featureMean, INDArray featureStd, INDArray labelMean, INDArray labelStd) {
        this.featureMean = featureMean;
        this.featureStd = featureStd;
        this.labelMean = labelMean;
        this.labelStd = labelStd;
        this.fitLabels = true;
    }

    private INDArray fit(INDArray theArray) {
        INDArray theMean, theStd;
        theMean = theArray.mean(0);
        theStd = theArray.std(0);
        theStd.addi(Nd4j.scalar(Nd4j.EPS_THRESHOLD));
        if (theStd.min(1) == Nd4j.scalar(Nd4j.EPS_THRESHOLD))
            logger.info("API_INFO: Std deviation found to be zero. Transform will round upto epsilon to avoid nans.");
        return Nd4j.vstack(theMean,theStd).dup();
    }

    private void runningFit(INDArray thenewArray, INDArray currentMeanStd, int runningTotal, boolean allDone) {
        int batchCount = thenewArray.size(0);
        INDArray currentMean = currentMeanStd.getRow(0);
        INDArray currentStd = currentMeanStd.getRow(1);
        if (!allDone) {
            // m_newM = m_oldM + (x - m_oldM)/m_n;
            INDArray xMinusMean = thenewArray.subRowVector(currentMean);
            INDArray newMean = currentMean.add(xMinusMean.sum(0).divi(runningTotal));
            // Using http://i.stanford.edu/pub/cstr/reports/cs/tr/79/773/CS-TR-79-773.pdf
{'fraction_non_alphanumeric': 0.10251595944423582, 'fraction_numerical': 0.016522718738265114, 'mean_word_length': 3.681019332161687, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6993597', 'n_tokens_mistral': 1848, 'n_tokens_neox': 1749, 'n_words': 331}