<commit_before>package edu.jhu.hltcoe.gridsearch.dmv;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

import org.apache.log4j.Logger;

import edu.jhu.hltcoe.data.DepTreebank;
import edu.jhu.hltcoe.gridsearch.Projector;
import edu.jhu.hltcoe.gridsearch.RelaxedSolution;
import edu.jhu.hltcoe.gridsearch.Solution;
import edu.jhu.hltcoe.gridsearch.dmv.BasicDmvProjector.DmvProjectorFactory;
import edu.jhu.hltcoe.gridsearch.dmv.BasicDmvProjector.DmvProjectorPrm;
import edu.jhu.hltcoe.model.dmv.CopyingDmvModelFactory;
import edu.jhu.hltcoe.model.dmv.DmvMStep;
import edu.jhu.hltcoe.model.dmv.DmvModel;
import edu.jhu.hltcoe.model.dmv.DmvModelFactory;
import edu.jhu.hltcoe.parse.DmvCkyParser;
import edu.jhu.hltcoe.parse.ViterbiParser;
import edu.jhu.hltcoe.train.DmvTrainCorpus;
import edu.jhu.hltcoe.train.DmvViterbiEMTrainer;
import edu.jhu.hltcoe.train.DmvViterbiEMTrainer.DmvViterbiEMTrainerPrm;
import edu.jhu.hltcoe.util.Prng;
import edu.jhu.hltcoe.util.Utilities;

public class ViterbiEmDmvProjector implements DmvProjector {

    public static class ViterbiEmDmvProjectorPrm implements DmvProjectorFactory {
        public double proportionViterbiImproveTreebank = 0.05;
        public double proportionViterbiImproveModel = 0.05;
        public DmvProjectorPrm dprojPrm = new DmvProjectorPrm();
        // EM Parameters for improving the projected solutions.
        public DmvViterbiEMTrainerPrm vemPrm = new DmvViterbiEMTrainerPrm();
        public ViterbiEmDmvProjectorPrm() {
            vemPrm.emPrm.iterations = 25;        
            vemPrm.emPrm.convergenceRatio = 0.99999;
            vemPrm.emPrm.numRestarts = 0;
            vemPrm.emPrm.timeoutSeconds = Double.POSITIVE_INFINITY;
            vemPrm.lambda = 0.1;
            vemPrm.evaluator = null;
        }
        @Override
        public Projector getInstance(DmvTrainCorpus corpus, DmvRelaxation relax) {
            return new ViterbiEmDmvProjector(this, corpus, relax);
        }
    }
    
    private final class DmvSolutionComparator implements Comparator<DmvSolution> {
        /**
         * This will only return nulls if there are no non-null entries
         */
        @Override
        public int compare(DmvSolution sol1, DmvSolution sol2) {
            if (sol1 == null && sol2 == null) {
                return 0;
            } else if (sol1 == null) {
                return 1;
            } else if (sol2 == null) {
                return -1;
            } else {
                return Double.compare(sol1.getScore(), sol2.getScore());
            }
        }
    }

    private static final Logger log = Logger.getLogger(ViterbiEmDmvProjector.class);

    private ViterbiEmDmvProjectorPrm prm;
    private BasicDmvProjector dmvProjector;
    private DmvTrainCorpus corpus;
    private DmvRelaxation relax;

    public ViterbiEmDmvProjector(ViterbiEmDmvProjectorPrm prm, DmvTrainCorpus corpus, DmvRelaxation dwRelax) {
        this.prm = prm;
        dmvProjector = new BasicDmvProjector(prm.dprojPrm, corpus);
        this.corpus = corpus;
        this.relax = dwRelax;
    }
    
    @Override
    public Solution getProjectedSolution(RelaxedSolution relaxSol) {
        return getProjectedDmvSolution((DmvRelaxedSolution) relaxSol);
    }

    public DmvSolution getProjectedDmvSolution(DmvRelaxedSolution relaxSol) {
        if (relaxSol == null) {
            throw new IllegalStateException("No relaxed solution cached.");
        }
        List<DmvSolution> solutions = new ArrayList<DmvSolution>();
        
        DmvSolution projectedSol = dmvProjector.getProjectedDmvSolution(relaxSol);
        // Add the null solution, so that the collection isn't empty.
        log.debug("Projected solution score: " + safeGetScore(projectedSol));
        solutions.add(projectedSol);
        if (projectedSol != null) {<commit_msg>Projecting with NORMALIZE and EUCLIDEAN<commit_after>package edu.jhu.hltcoe.gridsearch.dmv;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

import org.apache.log4j.Logger;

import edu.jhu.hltcoe.data.DepTreebank;
import edu.jhu.hltcoe.gridsearch.Projector;
import edu.jhu.hltcoe.gridsearch.RelaxedSolution;
import edu.jhu.hltcoe.gridsearch.Solution;
import edu.jhu.hltcoe.gridsearch.cpt.Projections.ProjectionsPrm.ProjectionType;
import edu.jhu.hltcoe.gridsearch.dmv.BasicDmvProjector.DmvProjectorFactory;
import edu.jhu.hltcoe.gridsearch.dmv.BasicDmvProjector.DmvProjectorPrm;
import edu.jhu.hltcoe.model.dmv.CopyingDmvModelFactory;
import edu.jhu.hltcoe.model.dmv.DmvMStep;
import edu.jhu.hltcoe.model.dmv.DmvModel;
import edu.jhu.hltcoe.model.dmv.DmvModelFactory;
import edu.jhu.hltcoe.parse.DmvCkyParser;
import edu.jhu.hltcoe.parse.ViterbiParser;
import edu.jhu.hltcoe.train.DmvTrainCorpus;
import edu.jhu.hltcoe.train.DmvViterbiEMTrainer;
import edu.jhu.hltcoe.train.DmvViterbiEMTrainer.DmvViterbiEMTrainerPrm;
import edu.jhu.hltcoe.util.Prng;
import edu.jhu.hltcoe.util.Utilities;

public class ViterbiEmDmvProjector implements DmvProjector {

    public static class ViterbiEmDmvProjectorPrm implements DmvProjectorFactory {
        public double proportionViterbiImproveTreebank = 0.05;
        public double proportionViterbiImproveModel = 0.05;
        public DmvProjectorPrm dprojPrm = new DmvProjectorPrm();
        // EM Parameters for improving the projected solutions.
        public DmvViterbiEMTrainerPrm vemPrm = new DmvViterbiEMTrainerPrm();
        public ViterbiEmDmvProjectorPrm() {
            vemPrm.emPrm.iterations = 25;        
            vemPrm.emPrm.convergenceRatio = 0.99999;
            vemPrm.emPrm.numRestarts = 0;
            vemPrm.emPrm.timeoutSeconds = Double.POSITIVE_INFINITY;
            vemPrm.lambda = 0.1;
            vemPrm.evaluator = null;
        }
        @Override
        public Projector getInstance(DmvTrainCorpus corpus, DmvRelaxation relax) {
            return new ViterbiEmDmvProjector(this, corpus, relax);
        }
    }
    
    private final class DmvSolutionComparator implements Comparator<DmvSolution> {
        /**
         * This will only return nulls if there are no non-null entries
         */
        @Override
        public int compare(DmvSolution sol1, DmvSolution sol2) {
            if (sol1 == null && sol2 == null) {
                return 0;
            } else if (sol1 == null) {
                return 1;
            } else if (sol2 == null) {
                return -1;
            } else {
                return Double.compare(sol1.getScore(), sol2.getScore());
            }
        }
    }

    private static final Logger log = Logger.getLogger(ViterbiEmDmvProjector.class);

    private ViterbiEmDmvProjectorPrm prm;
    private BasicDmvProjector normProjector;
    private BasicDmvProjector euclidProjector;
    private DmvTrainCorpus corpus;
    private DmvRelaxation relax;

    public ViterbiEmDmvProjector(ViterbiEmDmvProjectorPrm prm, DmvTrainCorpus corpus, DmvRelaxation dwRelax) {
        this.prm = prm;
        
        DmvProjectorPrm prm1 = new DmvProjectorPrm();
        prm1.rootBounds = prm.dprojPrm.rootBounds;
        prm1.projPrm.tempDir = prm.dprojPrm.projPrm.tempDir ;
        prm1.projPrm.lambdaSmoothing =  prm.dprojPrm.projPrm.lambdaSmoothing;
        prm1.projPrm.type =  ProjectionType.NORMALIZE;
        normProjector = new BasicDmvProjector(prm1, corpus);
        
        DmvProjectorPrm prm2 = new DmvProjectorPrm();
        prm2.rootBounds = prm.dprojPrm.rootBounds;
        prm2.projPrm.tempDir = prm.dprojPrm.projPrm.tempDir ;
        prm2.projPrm.lambdaSmoothing =  prm.dprojPrm.projPrm.lambdaSmoothing;
        prm2.projPrm.type =  ProjectionType.UNBOUNDED_MIN_EUCLIDEAN;
        euclidProjector = new BasicDmvProjector(prm2, corpus);

        this.corpus = corpus;
        this.relax = dwRelax;
    }
    
    @Override
    public Solution getProjectedSolution(RelaxedSolution relaxSol) {
        return getProjectedDmvSolution((DmvRelaxedSolution) relaxSol);
    }

    public DmvSolution getProjectedDmvSolution(DmvRelaxedSolution relaxSol) {
        if (relaxSol == null) {
            throw new IllegalStateException("No relaxed solution cached.");
        }
        List<DmvSolution> solutions = new ArrayList<DmvSolution>();

        // Add the null solution, so that the collection isn't empty.
        
        DmvSolution normSol = normProjector.getProjectedDmvSolution(relaxSol);
        log.debug("Normalized projected solution score: " + safeGetScore(normSol));
        solutions.add(normSol);
        
        DmvSolution euclidSol = euclidProjector.getProjectedDmvSolution(relaxSol);
        log.debug("Euclidean projected solution score: " + safeGetScore(euclidSol));
        solutions.add(euclidSol);
        
        DmvSolution projectedSol = Collections.max(solutions, new DmvSolutionComparator());
        if (projectedSol != null) {
{'fraction_non_alphanumeric': 0.08395835665509907, 'fraction_numerical': 0.007836113287809247, 'mean_word_length': 3.0170863309352516, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '979335', 'n_tokens_mistral': 3163, 'n_tokens_neox': 2969, 'n_words': 524}