<commit_before>package edu.neu.ccs.pyramid.classification.logistic_regression;

import edu.neu.ccs.pyramid.dataset.ClfDataSet;
import edu.neu.ccs.pyramid.dataset.DataSet;
import edu.neu.ccs.pyramid.dataset.ProbabilityMatrix;
import edu.neu.ccs.pyramid.optimization.*;
import edu.neu.ccs.pyramid.regression.linear_regression.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.mahout.math.DenseVector;
import org.apache.mahout.math.Vector;

import java.util.Arrays;
import java.util.stream.IntStream;

/**
 * Friedman, Jerome, <NAME>, and <NAME>.
 * "Regularization paths for generalized linear models via coordinate descent."
 * Journal of statistical software 33.1 (2010): 1.
 *
 * <NAME>, <NAME>, and <NAME>.
 * "An improved glmnet for l1-regularized logistic regression."
 * The Journal of Machine Learning Research 13.1 (2012): 1999-2030.
 *
 * <NAME> and <NAME>.
 * "Maxent Models, Conditional Estimation, and Optimization, without the Magic."
 *
 * Created by chengli on 2/24/15.
 */
public class ElasticNetLogisticTrainer {
    private static final Logger logger = LogManager.getLogger();
    private LogisticRegression logisticRegression;
    private DataSet dataSet;
    private int numClasses;
//    private int[] labels;
    // y_nl: number of datapoint and number of labels
    private double[][] targets;
    // instances weights
    private double[] weights;
    private double regularization;
    private double l1Ratio;
    // relative threshold
    private double epsilon;
    private Vector empiricalCounts;
    private Vector predictedCounts;
    private int numParameters;
    // size num classes * num data
    private double[][] probabilityMatrix;
    private Terminator terminator;
    private boolean lineSearch = true;

    public static Builder newBuilder(LogisticRegression logisticRegression, DataSet dataSet, int numClasses,
                                     double[][] targets, double[] weights) {
        return new Builder(logisticRegression, dataSet, numClasses, targets, weights);
    }

    public static Builder newBuilder(LogisticRegression logisticRegression, DataSet dataSet, int numClasses, double[][] targets) {
        return new Builder(logisticRegression, dataSet, numClasses, targets);
    }

    public static Builder newBuilder(LogisticRegression logisticRegression, DataSet dataSet, int numClasses, int[] labels){
        return new Builder(logisticRegression, dataSet, numClasses, labels);
    }

    public static Builder newBuilder(LogisticRegression logisticRegression, ClfDataSet dataSet){
        return new Builder(logisticRegression, dataSet);
    }

    public void optimize(){
        logisticRegression.setFeatureList(dataSet.getFeatureList());
//        logisticRegression.setLabelTranslator(dataSet.getLabelTranslator());

        while(true){
            iterate();
            if (terminator.shouldTerminate()){
                break;
            }
        }
    }

    public void iterate() {
        double[][] probs = new double[dataSet.getNumDataPoints()][numClasses];
        double[][] classScores = new double[dataSet.getNumDataPoints()][numClasses];
        IntStream.range(0, dataSet.getNumDataPoints()).parallel().forEach(i -> {
            probs[i] = logisticRegression.predictClassProbs(dataSet.getRow(i));
            classScores[i] = logisticRegression.predictClassScores(dataSet.getRow(i));
        });
        IntStream.range(0,numClasses).parallel().forEach(i -> optimizeOneClass(i, probs, classScores));
        if (lineSearch) {
            Weights oldWeights = logisticRegression.getWeights().deepCopy();
            Weights newWeights = logisticRegression.getWeights().deepCopy();
            // infer searchDirection
            Vector searchDirection = newWeights.getAllWeights().minus(oldWeights.getAllWeights());

            if (logger.isDebugEnabled()){
                logger.debug("norm of the search direction = " + searchDirection.norm(2));
            }
            // move back to starting point
            logisticRegression.getWeights().setWeightVector(oldWeights.getAllWeights());
            // this gradient doesn't include the penalty term, so it is only approximate
            Vector gradient = this.predictedCounts.minus(empiricalCounts).divide(dataSet.getNumDataPoints());
            lineSearch(searchDirection, gradient);
            updateClassProbMatrix();
            updatePredictedCounts();
        }
        terminator.add(getLoss());
    }

    private void optimizeOneClass(int classIndex, double[][] probs, double[][] classScores) {
        //create weighted least square problem
        int numDataPoints = dataSet.getNumDataPoints();
        double[] realLabels = new double[numDataPoints];
        double[] instanceWeights = new double[numDataPoints];
        IntStream.range(0,numDataPoints).parallel().forEach(i ->
        {
            // TODO: repeated calculations in following two steps.
            double prob = probs[i][classIndex];
            double classScore = classScores[i][classIndex];
            double y = targets[i][classIndex];
<commit_msg>bug fix in line search<commit_after>package edu.neu.ccs.pyramid.classification.logistic_regression;

import edu.neu.ccs.pyramid.dataset.ClfDataSet;
import edu.neu.ccs.pyramid.dataset.DataSet;
import edu.neu.ccs.pyramid.dataset.ProbabilityMatrix;
import edu.neu.ccs.pyramid.optimization.*;
import edu.neu.ccs.pyramid.regression.linear_regression.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.mahout.math.DenseVector;
import org.apache.mahout.math.Vector;

import java.util.Arrays;
import java.util.stream.IntStream;

/**
 * Friedman, Jerome, <NAME>, and <NAME>.
 * "Regularization paths for generalized linear models via coordinate descent."
 * Journal of statistical software 33.1 (2010): 1.
 *
 * Yuan, Guo-Xun, <NAME>, and <NAME>.
 * "An improved glmnet for l1-regularized logistic regression."
 * The Journal of Machine Learning Research 13.1 (2012): 1999-2030.
 *
 * <NAME> and <NAME>.
 * "Maxent Models, Conditional Estimation, and Optimization, without the Magic."
 *
 * Created by chengli on 2/24/15.
 */
public class ElasticNetLogisticTrainer {
    private static final Logger logger = LogManager.getLogger();
    private LogisticRegression logisticRegression;
    private DataSet dataSet;
    private int numClasses;
//    private int[] labels;
    // y_nl: number of datapoint and number of labels
    private double[][] targets;
    // instances weights
    private double[] weights;
    private double regularization;
    private double l1Ratio;
    // relative threshold
    private double epsilon;
    private Vector empiricalCounts;
    private Vector predictedCounts;
    private int numParameters;
    // size num classes * num data
    private double[][] probabilityMatrix;
    private Terminator terminator;
    private boolean lineSearch = true;

    public static Builder newBuilder(LogisticRegression logisticRegression, DataSet dataSet, int numClasses,
                                     double[][] targets, double[] weights) {
        return new Builder(logisticRegression, dataSet, numClasses, targets, weights);
    }

    public static Builder newBuilder(LogisticRegression logisticRegression, DataSet dataSet, int numClasses, double[][] targets) {
        return new Builder(logisticRegression, dataSet, numClasses, targets);
    }

    public static Builder newBuilder(LogisticRegression logisticRegression, DataSet dataSet, int numClasses, int[] labels){
        return new Builder(logisticRegression, dataSet, numClasses, labels);
    }

    public static Builder newBuilder(LogisticRegression logisticRegression, ClfDataSet dataSet){
        return new Builder(logisticRegression, dataSet);
    }

    public void optimize(){
        logisticRegression.setFeatureList(dataSet.getFeatureList());
//        logisticRegression.setLabelTranslator(dataSet.getLabelTranslator());

        while(true){
            iterate();
            if (terminator.shouldTerminate()){
                break;
            }
        }
    }

    public void iterate() {
        double[][] probs = new double[dataSet.getNumDataPoints()][numClasses];
        double[][] classScores = new double[dataSet.getNumDataPoints()][numClasses];
        IntStream.range(0, dataSet.getNumDataPoints()).parallel().forEach(i -> {
            probs[i] = logisticRegression.predictClassProbs(dataSet.getRow(i));
            classScores[i] = logisticRegression.predictClassScores(dataSet.getRow(i));
        });
        Weights oldWeights = logisticRegression.getWeights().deepCopy();
        IntStream.range(0,numClasses).parallel().forEach(i -> optimizeOneClass(i, probs, classScores));
        if (lineSearch) {
            Weights newWeights = logisticRegression.getWeights().deepCopy();
            // infer searchDirection
            Vector searchDirection = newWeights.getAllWeights().minus(oldWeights.getAllWeights());

            if (logger.isDebugEnabled()){
                logger.debug("norm of the search direction = " + searchDirection.norm(2));
            }
            // move back to starting point
            logisticRegression.getWeights().setWeightVector(oldWeights.getAllWeights());
            // this gradient doesn't include the penalty term, so it is only approximate
            Vector gradient = this.predictedCounts.minus(empiricalCounts).divide(dataSet.getNumDataPoints());
            lineSearch(searchDirection, gradient);
            updateClassProbMatrix();
            updatePredictedCounts();
        }
        terminator.add(getLoss());
    }

    private void optimizeOneClass(int classIndex, double[][] probs, double[][] classScores) {
        //create weighted least square problem
        int numDataPoints = dataSet.getNumDataPoints();
        double[] realLabels = new double[numDataPoints];
        double[] instanceWeights = new double[numDataPoints];
        IntStream.range(0,numDataPoints).parallel().forEach(i ->
        {
            // TODO: repeated calculations in following two steps.
            double prob = probs[i][classIndex];
            double classScore = classScores[i][classIndex];
            double y = targets[i][classIndex];

{'fraction_non_alphanumeric': 0.10186625194401244, 'fraction_numerical': 0.006998444790046656, 'mean_word_length': 3.267523849025301, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6749059', 'n_tokens_mistral': 2885, 'n_tokens_neox': 2759, 'n_words': 681}