<commit_before>package edu.jhu.hltcoe.gridsearch.dr;

import ilog.concert.IloException;
import ilog.concert.IloLPMatrix;

import java.util.Collection;

import no.uib.cipr.matrix.sparse.longs.FastSparseLVector;

import org.apache.log4j.Logger;

import cern.colt.matrix.tdouble.DoubleFactory2D;
import cern.colt.matrix.tdouble.impl.DenseDoubleMatrix2D;
import cern.colt.matrix.tdouble.impl.SparseCCDoubleMatrix2D;
import cern.jet.random.Gamma;
import cern.jet.random.tdouble.Beta;
import edu.jhu.hltcoe.lp.CcLpConstraints;
import edu.jhu.hltcoe.lp.FactorList;
import edu.jhu.hltcoe.lp.LpRows;
import edu.jhu.hltcoe.lp.FactorBuilder.Factor;
import edu.jhu.hltcoe.util.Pair;
import edu.jhu.hltcoe.util.Prng;
import edu.jhu.hltcoe.util.SafeCast;
import edu.jhu.hltcoe.util.Utilities;
import edu.jhu.hltcoe.util.cplex.CplexUtils;
import edu.jhu.hltcoe.util.tuple.OrderedPair;
import edu.jhu.hltcoe.util.tuple.PairSampler;

/**
 * Reduces the dimensionality of a linear program.
 * 
 * @author mgormley
 * 
 */
public class DimReducer {
    
    public static class DimReducerPrm {
        public int drMaxCons = Integer.MAX_VALUE;
        public boolean setNames = true;
        public boolean renormalize = true;
        public boolean includeBounds = false;
        /**
         * The delta for checking zero in the sampled matrix.
         */
        public double samplingZeroCutoff = 1e-2;
        public SamplingDistribution dist = SamplingDistribution.UNIFORM;
        /**
         * Parameter for Beta and Gamma distribution.
         */
        public double alpha = 0.01;
        /**
         * Parameter for Beta and Gamma distribution.
         * For Gamma, this is the rate parameter beta = 1 / lambda. 
         */
        public double beta = 100;
        /**
         * The delta for deciding what a zero is in the projected matrix, SA.
         * Care should be taken when changing this since it could produce an infeasibility.
         */
        public double multZeroDelta = 1e-13;
        public int maxNonZeros = Integer.MAX_VALUE;
        /**
         * This parameter uses the identity matrix for S, overriding all other
         * projection preferences. This is just used as a sanity check.
         */
        public boolean useIdentityMatrix = false;
        /**
         * Method of converting the input constraints (which are defined as Ax
         * <= b and Dx == d) into a set of constraints for projection.
         */
        public ConstraintConversion conversion = ConstraintConversion.EQ_TO_LEQ_PAIR;
    }

    /**
     * Enumeration of sampling distributions for populating the projection
     * matrix.
     * 
     * Note that these all sample from the positive reals, since a negative
     * number would flip the inequality.
     * 
     * TODO: If we switch to all equality constraints (Ax = b), we could use
     * sampling distributions with negative numbers.
     */
    public enum SamplingDistribution {
        UNIFORM, BETA, GAMMA, ALL_ONES
    }
    
    public enum ConstraintConversion {
        EQ_TO_LEQ_PAIR, SEPARATE_EQ_AND_LEQ
    }
    
    private static final Logger log = Logger.getLogger(DimReducer.class);

    private DimReducerPrm prm;

    public DimReducer(DimReducerPrm prm) {
        this.prm = prm;
    }

    /**
     * Reduces the dimensionality of a matrix representing a set of linear
     * constraints.
     * 
     * @param origMatrix
     *            Input matrix.
     * @param drMatrix
     *            Output matrix.
     * @throws IloException
     */
    public void reduceDimensionality(IloLPMatrix origMatrix, IloLPMatrix drMatrix) throws IloException {
        if (prm.drMaxCons >= origMatrix.getNrows() && !prm.useIdentityMatrix) {
            if (prm.drMaxCons != Integer.MAX_VALUE) {
                log.warn(String.format("Parameter drMaxCons set to %d but matrix only has %d rows. " + 
                        "Skipping dimentionality reduction.", prm.drMaxCons, origMatrix.getNrows()));
            }
            // Do nothing.
            return;
        }
        
        // Add IloNumVars to lower dimensional matrix.
        int nCols = origMatrix.getNcols(); //TODO: if we add variables this needs to change.
        drMatrix.addCols(origMatrix.getNumVars());

        // Get the rows of the matrix (but not the bounds).
        FactorList factors;
        if (prm.includeBounds ) {
            factors = FactorList.getFactors(origMatrix, false);
        } else {
            // Get the rows of the matrix (but not the bounds).
            factors = FactorList.getRowFactors(origMatrix);
        }

        // Renormalize the rows.
        if (prm.renormalize) {
            factors.renormalize();
        }

        if (prm.conversion == ConstraintConversion.EQ_TO_LEQ_PAIR) {
            // Convert the original matrix to Ax <= b form.
            CcLpConstraints lc = getFactorsAsLeqConstraints(factors, nCols);
            projectAndAddConstraints(lc, drMatrix, prm.drMaxCons);
        } else if (prm.conversion == ConstraintConversion.SEPARATE_EQ_AND_LEQ) {
            // Project the equality and inequality constraints separately.
            Pair<CcLpConstraints,CcLpConstraints> pair = getFactorsAsEqAndLeqConstraints(factors, nCols);
            CcLpConstraints eqLc = pair.get1();
            CcLpConstraints leqLc = pair.get2();
            double propEq = (double) eqLc.getNumRows() / (eqLc.getNumRows() + leqLc.getNumRows());
            log.info("Proportion equality constraints: " + propEq);
            projectAndAddConstraints(eqLc, drMatrix, SafeCast.safeDoubleToInt(propEq * prm.drMaxCons));
            projectAndAddConstraints(leqLc, drMatrix, SafeCast.safeDoubleToInt((1.0 - propEq) * prm.drMaxCons));
        } else {
            throw new RuntimeException("Unhandled constraint conversion method: " + prm.conversion);
        }
        log.debug("Number of nonzeros in orig matrix: " + origMatrix.getNNZs());
        log.debug("Number of nonzeros in DR matrix: " + drMatrix.getNNZs());
    }

    /**
     * Project the constraints down to a lower dimensional space.
     * 
     * @param lc Input constraints.
     * @param drMatrix Output matrix.
     * @param drMaxCons Maximum number of constraints to form.<commit_msg>Rounding ints<commit_after>package edu.jhu.hltcoe.gridsearch.dr;

import ilog.concert.IloException;
import ilog.concert.IloLPMatrix;

import java.util.Collection;

import no.uib.cipr.matrix.sparse.longs.FastSparseLVector;

import org.apache.log4j.Logger;

import cern.colt.matrix.tdouble.DoubleFactory2D;
import cern.colt.matrix.tdouble.impl.DenseDoubleMatrix2D;
import cern.colt.matrix.tdouble.impl.SparseCCDoubleMatrix2D;
import cern.jet.random.Gamma;
import cern.jet.random.tdouble.Beta;
import edu.jhu.hltcoe.lp.CcLpConstraints;
import edu.jhu.hltcoe.lp.FactorList;
import edu.jhu.hltcoe.lp.LpRows;
import edu.jhu.hltcoe.lp.FactorBuilder.Factor;
import edu.jhu.hltcoe.util.Pair;
import edu.jhu.hltcoe.util.Prng;
import edu.jhu.hltcoe.util.SafeCast;
import edu.jhu.hltcoe.util.Utilities;
import edu.jhu.hltcoe.util.cplex.CplexUtils;
import edu.jhu.hltcoe.util.tuple.OrderedPair;
import edu.jhu.hltcoe.util.tuple.PairSampler;

/**
 * Reduces the dimensionality of a linear program.
 * 
 * @author mgormley
 * 
 */
public class DimReducer {
    
    public static class DimReducerPrm {
        public int drMaxCons = Integer.MAX_VALUE;
        public boolean setNames = true;
        public boolean renormalize = true;
        public boolean includeBounds = false;
        /**
         * The delta for checking zero in the sampled matrix.
         */
        public double samplingZeroCutoff = 1e-2;
        public SamplingDistribution dist = SamplingDistribution.UNIFORM;
        /**
         * Parameter for Beta and Gamma distribution.
         */
        public double alpha = 0.01;
        /**
         * Parameter for Beta and Gamma distribution.
         * For Gamma, this is the rate parameter beta = 1 / lambda. 
         */
        public double beta = 100;
        /**
         * The delta for deciding what a zero is in the projected matrix, SA.
         * Care should be taken when changing this since it could produce an infeasibility.
         */
        public double multZeroDelta = 1e-13;
        public int maxNonZeros = Integer.MAX_VALUE;
        /**
         * This parameter uses the identity matrix for S, overriding all other
         * projection preferences. This is just used as a sanity check.
         */
        public boolean useIdentityMatrix = false;
        /**
         * Method of converting the input constraints (which are defined as Ax
         * <= b and Dx == d) into a set of constraints for projection.
         */
        public ConstraintConversion conversion = ConstraintConversion.EQ_TO_LEQ_PAIR;
    }

    /**
     * Enumeration of sampling distributions for populating the projection
     * matrix.
     * 
     * Note that these all sample from the positive reals, since a negative
     * number would flip the inequality.
     * 
     * TODO: If we switch to all equality constraints (Ax = b), we could use
     * sampling distributions with negative numbers.
     */
    public enum SamplingDistribution {
        UNIFORM, BETA, GAMMA, ALL_ONES
    }
    
    public enum ConstraintConversion {
        EQ_TO_LEQ_PAIR, SEPARATE_EQ_AND_LEQ
    }
    
    private static final Logger log = Logger.getLogger(DimReducer.class);

    private DimReducerPrm prm;

    public DimReducer(DimReducerPrm prm) {
        this.prm = prm;
    }

    /**
     * Reduces the dimensionality of a matrix representing a set of linear
     * constraints.
     * 
     * @param origMatrix
     *            Input matrix.
     * @param drMatrix
     *            Output matrix.
     * @throws IloException
     */
    public void reduceDimensionality(IloLPMatrix origMatrix, IloLPMatrix drMatrix) throws IloException {
        if (prm.drMaxCons >= origMatrix.getNrows() && !prm.useIdentityMatrix) {
            if (prm.drMaxCons != Integer.MAX_VALUE) {
                log.warn(String.format("Parameter drMaxCons set to %d but matrix only has %d rows. " + 
                        "Skipping dimentionality reduction.", prm.drMaxCons, origMatrix.getNrows()));
            }
            // Do nothing.
            return;
        }
        
        // Add IloNumVars to lower dimensional matrix.
        int nCols = origMatrix.getNcols(); //TODO: if we add variables this needs to change.
        drMatrix.addCols(origMatrix.getNumVars());

        // Get the rows of the matrix (but not the bounds).
        FactorList factors;
        if (prm.includeBounds ) {
            factors = FactorList.getFactors(origMatrix, false);
        } else {
            // Get the rows of the matrix (but not the bounds).
            factors = FactorList.getRowFactors(origMatrix);
        }

        // Renormalize the rows.
        if (prm.renormalize) {
            factors.renormalize();
        }

        if (prm.conversion == ConstraintConversion.EQ_TO_LEQ_PAIR) {
            // Convert the original matrix to Ax <= b form.
            CcLpConstraints lc = getFactorsAsLeqConstraints(factors, nCols);
            projectAndAddConstraints(lc, drMatrix, prm.drMaxCons);
        } else if (prm.conversion == ConstraintConversion.SEPARATE_EQ_AND_LEQ) {
            // Project the equality and inequality constraints separately.
            Pair<CcLpConstraints,CcLpConstraints> pair = getFactorsAsEqAndLeqConstraints(factors, nCols);
            CcLpConstraints eqLc = pair.get1();
            CcLpConstraints leqLc = pair.get2();
            double propEq = (double) eqLc.getNumRows() / (eqLc.getNumRows() + leqLc.getNumRows());
            log.info("Proportion equality constraints: " + propEq);
            projectAndAddConstraints(eqLc, drMatrix, SafeCast.safeLongToInt(Math.round(propEq * prm.drMaxCons)));
            projectAndAddConstraints(leqLc, drMatrix, SafeCast.safeLongToInt(Math.round((1.0 - propEq) * prm.drMaxCons)));
        } else {
            throw new RuntimeException("Unhandled constraint conversion method: " + prm.conversion);
        }
        log.debug("Number of nonzeros in orig matrix: " + origMatrix.getNNZs());
        log.debug("Number of nonzeros in DR matrix: " + drMatrix.getNNZs());
    }

    /**
     * Project the constraints down to a lower dimensional space.
     * 
     * @param lc Input constraints.
     * @param drMatrix Output matrix.
     * @param drMaxCons Maximum number of constraints to form.
{'fraction_non_alphanumeric': 0.08729016786570744, 'fraction_numerical': 0.0031974420463629096, 'mean_word_length': 2.4220459518599564, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4662604', 'n_tokens_mistral': 3799, 'n_tokens_neox': 3660, 'n_words': 988}