<commit_before>package edu.jhu.hltcoe.train;

import java.io.File;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;

import edu.jhu.hltcoe.data.DepTreebank;
import edu.jhu.hltcoe.eval.DependencyParserEvaluator;
import edu.jhu.hltcoe.gridsearch.BfsComparator;
import edu.jhu.hltcoe.gridsearch.DfsBfcComparator;
import edu.jhu.hltcoe.gridsearch.DfsRandChildAtDepthNodeOrderer;
import edu.jhu.hltcoe.gridsearch.DfsRandWalkNodeOrderer;
import edu.jhu.hltcoe.gridsearch.DmvLazyBranchAndBoundSolver;
import edu.jhu.hltcoe.gridsearch.LazyBranchAndBoundSolver;
import edu.jhu.hltcoe.gridsearch.NodeOrderer;
import edu.jhu.hltcoe.gridsearch.PqNodeOrderer;
import edu.jhu.hltcoe.gridsearch.cpt.BasicCptBoundsDeltaFactory;
import edu.jhu.hltcoe.gridsearch.cpt.CptBoundsDeltaFactory;
import edu.jhu.hltcoe.gridsearch.cpt.FullStrongVariableSelector;
import edu.jhu.hltcoe.gridsearch.cpt.MidpointVarSplitter;
import edu.jhu.hltcoe.gridsearch.cpt.PseudocostVariableSelector;
import edu.jhu.hltcoe.gridsearch.cpt.RandomVariableSelector;
import edu.jhu.hltcoe.gridsearch.cpt.RegretVariableSelector;
import edu.jhu.hltcoe.gridsearch.cpt.VariableSelector;
import edu.jhu.hltcoe.gridsearch.cpt.VariableSplitter;
import edu.jhu.hltcoe.gridsearch.cpt.LpSumToOneBuilder.CutCountComputer;
import edu.jhu.hltcoe.gridsearch.cpt.LpSumToOneBuilder.LpStoBuilderPrm;
import edu.jhu.hltcoe.gridsearch.cpt.MidpointVarSplitter.MidpointChoice;
import edu.jhu.hltcoe.gridsearch.dmv.BnBDmvTrainer;
import edu.jhu.hltcoe.gridsearch.dmv.DmvDantzigWolfeRelaxation;
import edu.jhu.hltcoe.gridsearch.dmv.DmvRelaxation;
import edu.jhu.hltcoe.gridsearch.dmv.DmvRltRelaxation;
import edu.jhu.hltcoe.gridsearch.dmv.ResDmvDantzigWolfeRelaxation;
import edu.jhu.hltcoe.gridsearch.dmv.DmvDantzigWolfeRelaxation.DmvDwRelaxPrm;
import edu.jhu.hltcoe.gridsearch.dmv.DmvRltRelaxation.DmvRltRelaxPrm;
import edu.jhu.hltcoe.gridsearch.dmv.ResDmvDantzigWolfeRelaxation.ResDmvDwRelaxPrm;
import edu.jhu.hltcoe.gridsearch.rlt.Rlt.RltPrm;
import edu.jhu.hltcoe.gridsearch.rlt.filter.RandPropRltRowFilter;
import edu.jhu.hltcoe.ilp.IlpSolverFactory;
import edu.jhu.hltcoe.ilp.IlpSolverFactory.IlpSolverId;
import edu.jhu.hltcoe.lp.CplexPrm;
import edu.jhu.hltcoe.model.FixableModelFactory;
import edu.jhu.hltcoe.model.ModelFactory;
import edu.jhu.hltcoe.model.dmv.DmvMStep;
import edu.jhu.hltcoe.model.dmv.DmvModel;
import edu.jhu.hltcoe.model.dmv.RandomDmvModelFactory;
import edu.jhu.hltcoe.model.dmv.SupervisedDmvModelFactory;
import edu.jhu.hltcoe.model.dmv.UniformDmvModelFactory;
import edu.jhu.hltcoe.parse.DeltaGenerator;
import edu.jhu.hltcoe.parse.DmvCkyParser;
import edu.jhu.hltcoe.parse.FactorDeltaGenerator;
import edu.jhu.hltcoe.parse.FixedIntervalDeltaGenerator;
import edu.jhu.hltcoe.parse.IlpFormulation;
import edu.jhu.hltcoe.parse.IlpViterbiParser;
import edu.jhu.hltcoe.parse.IlpViterbiParserWithDeltas;
import edu.jhu.hltcoe.parse.IlpViterbiSentenceParser;
import edu.jhu.hltcoe.parse.InitializedIlpViterbiParserWithDeltas;
import edu.jhu.hltcoe.parse.ViterbiParser;
import edu.jhu.hltcoe.util.Command;

/**
 * TODO: Consider switching to annotations
 * http://docs.oracle.com/javase/tutorial/java/javaOO/annotations.html
 * http://tutorials.jenkov.com/java-reflection/annotations.html
 * 
 * @author mgormley
 * 
 */
public class TrainerFactory {

    private static ViterbiParser evalParser = null;

    public static void addOptions(Options options) {
        options.addOption("a", "algorithm", true, "Inference algorithm");
        options.addOption("i", "iterations", true, "Number of iterations");
        options.addOption("i", "convergenceRatio", true, "Convergence ratio");
        options.addOption("nr", "numRestarts", true, "Number of random restarts");
        options.addOption("m", "model", true, "Model");
        options.addOption("p", "parser", true, "Parser");
        options.addOption("d", "initWeights", true, "Method for initializing the weights [uniform, random, supervised]");
        options.addOption("d", "deltaGenerator", true, "Delta generator");
        options.addOption("in", "interval", true, "Only for fixed-interval delta generator");
        options.addOption("fa", "factor", true, "Only for factor delta generator");
        options.addOption("nps", "numPerSide", true, "For symmetric delta generators");
        options.addOption("f", "formulation", true, "ILP formulation for parsing");
        options.addOption("l", "lambda", true, "Value for add-lambda smoothing");
        options.addOption("t", "threads", true, "Number of threads for parallel impl");
        options.addOption("ilp", "ilpSolver", true, "The ILP solver to use " + IlpSolverId.getIdList());
        options.addOption("ilpwmm", "ilpWorkMemMegs", true, "The working memory allotted for the ILP solver in megabytes");
        options.addOption("e", "epsilon", true, "Suboptimality convergence criterion for branch-and-bound");
        options.addOption("vse", "varSelection", true, "Variable selection strategy for branching [full,regret,rand-uniform,rand-weighted]");
        options.addOption("vsp", "varSplit", true, "Variable splitting strategy for branching [half-prob, half-logprob]");
        options.addOption("no", "nodeOrder", true, "Strategy for node selection [bfs, dfs]");
        options.addOption("rx", "relaxation", true, "Relaxation [dw,dw-res,rlt]");
        options.addOption("rx", "maxSimplexIterations", true, "(D-W only) The maximum number of simplex iterations");
        options.addOption("rx", "maxDwIterations", true, "(D-W only) The maximum number of dantzig-wolfe algorithm iterations");
        options.addOption("rx", "maxSetSizeToConstrain", true, "(D-W only) The maximum size of sets to contrain to be <= 1.0");
        options.addOption("rx", "maxCutRounds", true, "(D-W only) The maximum number of rounds to add cuts");
        options.addOption("rx", "minSumForCuts", true, "(D-W only) The minimum threshold at which to stop adding cuts");
        options.addOption("dwt", "dwTempDir", true, "(D-W only) For testing only. The temporary directory to which CPLEX files should be written");
        options.addOption("op", "offsetProb", true, "How much to offset the bounds in probability space from the initial bounds point");
        options.addOption("op", "probOfSkipCm", true, "The probability of not bounding a particular variable");
        options.addOption("op", "timeoutSeconds", true, "The timeout in seconds for training run");
        options.addOption("op", "bnbTimeoutSeconds", true, "[Viterbi-B&B only] The timeout in seconds for branch-and-bound");
        options.addOption("df", "disableFathoming", true, "Disables fathoming in branch-and-bound");
        options.addOption("eo", "envelopeOnly", true, "Whether to use only the convex/concave envelope for the RLT relaxation");
        options.addOption("eo", "rltFilter", true, "RLT filter type [obj-var, prop]");
        options.addOption("eo", "rltInitProp", true, "(prop only) Proportion of initial rows to accept.");
        options.addOption("eo", "rltCutProp", true, "(prop only) Proportion of cut rows to accept.");
    }

    public static Object getTrainer(CommandLine cmd, DepTreebank trainTreebank, DmvModel trueModel) throws ParseException {

        final String algorithm = Command.getOptionValue(cmd, "algorithm", "viterbi");
        final int iterations = Command.getOptionValue(cmd, "iterations", 10);
        final double convergenceRatio = Command.getOptionValue(cmd, "convergenceRatio", 0.99999);
        final int numRestarts = Command.getOptionValue(cmd, "numRestarts", 0);
        final String modelName = Command.getOptionValue(cmd, "model", "dmv");
        final String parserName = Command.getOptionValue(cmd, "parser", "ilp-sentence");
        final String initWeights = Command.getOptionValue(cmd, "initWeights", "uniform");
        final String deltaGenerator = Command.getOptionValue(cmd, "deltaGenerator", "fixed");
        final double interval = Command.getOptionValue(cmd, "interval", 0.01);
        final double factor = Command.getOptionValue(cmd, "factor", 1.1);
        final int numPerSide = Command.getOptionValue(cmd, "numPerSide", 2);
        final IlpFormulation formulation = getOptionValue(cmd, "formulation", IlpFormulation.DP_PROJ);
        final double lambda = Command.getOptionValue(cmd, "lambda", 0.1);
        final int numThreads = Command.getOptionValue(cmd, "threads", 2);
        final String ilpSolver = Command.getOptionValue(cmd, "ilpSolver", "cplex");
        final double ilpWorkMemMegs = Command.getOptionValue(cmd, "ilpWorkMemMegs", 512.0);
        final double epsilon = Command.getOptionValue(cmd, "epsilon", 0.1);
        final String varSelection = Command.getOptionValue(cmd, "varSelection", "regret");
        final String varSplit = Command.getOptionValue(cmd, "varSplit", "half-prob");
        final String nodeOrder = Command.getOptionValue(cmd, "nodeOrder", "bfs");
        final String relaxation = Command.getOptionValue(cmd, "relaxation", "dw");
        final int maxSimplexIterations = Command.getOptionValue(cmd, "maxSimplexIterations", 2100000000);
        final int maxDwIterations = Command.getOptionValue(cmd, "maxDwIterations", 1000);
        final int maxSetSizeToConstrain = Command.getOptionValue(cmd, "maxSetSizeToConstrain", 2);
        final int maxCutRounds = Command.getOptionValue(cmd, "maxCutRounds", 100);
        final double minSumForCuts = Command.getOptionValue(cmd, "minSumForCuts", 1.01);
        final String dwTempDir = Command.getOptionValue(cmd, "dwTempDir", "");
        final double offsetProb = Command.getOptionValue(cmd, "offsetProb", 1.0);
        final double probOfSkipCm = Command.getOptionValue(cmd, "probOfSkipCm", 0.0);
        final double timeoutSeconds = Command.getOptionValue(cmd, "timeoutSeconds", Double.POSITIVE_INFINITY);
        final double bnbTimeoutSeconds = Command.getOptionValue(cmd, "bnbTimeoutSeconds", Double.POSITIVE_INFINITY);
        final boolean disableFathoming = Command.getOptionValue(cmd, "disableFathoming", false);
        final boolean envelopeOnly = Command.getOptionValue(cmd, "envelopeOnly", true);
        final String rltFilter = Command.getOptionValue(cmd, "rltFilter", "obj-var");
        final double rltInitProp = Command.getOptionValue(cmd, "rltInitProp", 0.1);
        final double rltCutProp = Command.getOptionValue(cmd, "rltCutProp", 0.1);
        
        if (!modelName.equals("dmv")) {
            throw new ParseException("Model not supported: " + modelName);
        }

        double simplexTimeout = algorithm.equals("viterbi-bnb") ? bnbTimeoutSeconds : timeoutSeconds;
        CplexPrm cplexPrm = new CplexPrm(ilpWorkMemMegs, numThreads, maxSimplexIterations, simplexTimeout);

        LpStoBuilderPrm stoPrm = new LpStoBuilderPrm();
        stoPrm.initCutCountComp = new CutCountComputer();
        stoPrm.maxSetSizeToConstrain = maxSetSizeToConstrain;
        stoPrm.minSumForCuts = minSumForCuts;
        
        DmvRelaxation relax = null;
        if (cmd.hasOption("relaxOnly") || algorithm.equals("bnb") || algorithm.equals("viterbi-bnb")) {
            File dwTemp = dwTempDir.equals("") ? null : new File(dwTempDir);
            if (relaxation.equals("dw")) {
                DmvDwRelaxPrm dwPrm = new DmvDwRelaxPrm();
                dwPrm.tempDir = dwTemp;
                dwPrm.maxCutRounds = maxCutRounds;
                dwPrm.cplexPrm = cplexPrm;
                dwPrm.maxDwIterations = maxDwIterations;
                dwPrm.stoPrm = stoPrm;
                relax = new DmvDantzigWolfeRelaxation(dwPrm);
            } else if (relaxation.equals("dw-res")) {
                ResDmvDwRelaxPrm dwPrm = new ResDmvDwRelaxPrm();
                dwPrm.tempDir = dwTemp;
                dwPrm.maxCutRounds = maxCutRounds;
                dwPrm.cplexPrm = cplexPrm;
                dwPrm.maxDwIterations = maxDwIterations;
                relax = new ResDmvDantzigWolfeRelaxation(dwPrm);
            } else if (relaxation.equals("rlt")) {
                RltPrm rltPrm = new RltPrm();
                rltPrm.nameRltVarsAndCons = false;
                rltPrm.envelopeOnly = envelopeOnly;
                    
                DmvRltRelaxPrm rrPrm = new DmvRltRelaxPrm();
                rrPrm.tempDir = dwTemp;
                rrPrm.maxCutRounds = maxCutRounds;
                rrPrm.cplexPrm = cplexPrm;
                rrPrm.rltPrm = rltPrm;
                rrPrm.stoPrm = stoPrm;
                
                rrPrm.objVarFilter = false;
                if (rltFilter.equals("obj-var")) {
                    rrPrm.objVarFilter = true;
                    rltPrm.factorFilter = null;
                    rltPrm.rowFilter = null;
                } else if (rltFilter.equals("prop")) {
                    rltPrm.rowFilter = new RandPropRltRowFilter(rltInitProp, rltCutProp);
                } else {
                    throw new ParseException("RLT filter type not supported: " + rltFilter);
                }
                relax = new DmvRltRelaxation(rrPrm);
            } else {
                throw new ParseException("Relaxation not supported: " + relaxation);
            }
            if (cmd.hasOption("relaxOnly")) {
                return relax;
            }
        }

        evalParser = new DmvCkyParser();
        DependencyParserEvaluator parserEvaluator = new DependencyParserEvaluator(evalParser, trainTreebank, "train"); 
        <commit_msg>adding flag for RLT names on/off<commit_after>package edu.jhu.hltcoe.train;

import java.io.File;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;

import edu.jhu.hltcoe.data.DepTreebank;
import edu.jhu.hltcoe.eval.DependencyParserEvaluator;
import edu.jhu.hltcoe.gridsearch.BfsComparator;
import edu.jhu.hltcoe.gridsearch.DfsBfcComparator;
import edu.jhu.hltcoe.gridsearch.DfsRandChildAtDepthNodeOrderer;
import edu.jhu.hltcoe.gridsearch.DfsRandWalkNodeOrderer;
import edu.jhu.hltcoe.gridsearch.DmvLazyBranchAndBoundSolver;
import edu.jhu.hltcoe.gridsearch.LazyBranchAndBoundSolver;
import edu.jhu.hltcoe.gridsearch.NodeOrderer;
import edu.jhu.hltcoe.gridsearch.PqNodeOrderer;
import edu.jhu.hltcoe.gridsearch.cpt.BasicCptBoundsDeltaFactory;
import edu.jhu.hltcoe.gridsearch.cpt.CptBoundsDeltaFactory;
import edu.jhu.hltcoe.gridsearch.cpt.FullStrongVariableSelector;
import edu.jhu.hltcoe.gridsearch.cpt.MidpointVarSplitter;
import edu.jhu.hltcoe.gridsearch.cpt.PseudocostVariableSelector;
import edu.jhu.hltcoe.gridsearch.cpt.RandomVariableSelector;
import edu.jhu.hltcoe.gridsearch.cpt.RegretVariableSelector;
import edu.jhu.hltcoe.gridsearch.cpt.VariableSelector;
import edu.jhu.hltcoe.gridsearch.cpt.VariableSplitter;
import edu.jhu.hltcoe.gridsearch.cpt.LpSumToOneBuilder.CutCountComputer;
import edu.jhu.hltcoe.gridsearch.cpt.LpSumToOneBuilder.LpStoBuilderPrm;
import edu.jhu.hltcoe.gridsearch.cpt.MidpointVarSplitter.MidpointChoice;
import edu.jhu.hltcoe.gridsearch.dmv.BnBDmvTrainer;
import edu.jhu.hltcoe.gridsearch.dmv.DmvDantzigWolfeRelaxation;
import edu.jhu.hltcoe.gridsearch.dmv.DmvRelaxation;
import edu.jhu.hltcoe.gridsearch.dmv.DmvRltRelaxation;
import edu.jhu.hltcoe.gridsearch.dmv.ResDmvDantzigWolfeRelaxation;
import edu.jhu.hltcoe.gridsearch.dmv.DmvDantzigWolfeRelaxation.DmvDwRelaxPrm;
import edu.jhu.hltcoe.gridsearch.dmv.DmvRltRelaxation.DmvRltRelaxPrm;
import edu.jhu.hltcoe.gridsearch.dmv.ResDmvDantzigWolfeRelaxation.ResDmvDwRelaxPrm;
import edu.jhu.hltcoe.gridsearch.rlt.Rlt.RltPrm;
import edu.jhu.hltcoe.gridsearch.rlt.filter.RandPropRltRowFilter;
import edu.jhu.hltcoe.ilp.IlpSolverFactory;
import edu.jhu.hltcoe.ilp.IlpSolverFactory.IlpSolverId;
import edu.jhu.hltcoe.lp.CplexPrm;
import edu.jhu.hltcoe.model.FixableModelFactory;
import edu.jhu.hltcoe.model.ModelFactory;
import edu.jhu.hltcoe.model.dmv.DmvMStep;
import edu.jhu.hltcoe.model.dmv.DmvModel;
import edu.jhu.hltcoe.model.dmv.RandomDmvModelFactory;
import edu.jhu.hltcoe.model.dmv.SupervisedDmvModelFactory;
import edu.jhu.hltcoe.model.dmv.UniformDmvModelFactory;
import edu.jhu.hltcoe.parse.DeltaGenerator;
import edu.jhu.hltcoe.parse.DmvCkyParser;
import edu.jhu.hltcoe.parse.FactorDeltaGenerator;
import edu.jhu.hltcoe.parse.FixedIntervalDeltaGenerator;
import edu.jhu.hltcoe.parse.IlpFormulation;
import edu.jhu.hltcoe.parse.IlpViterbiParser;
import edu.jhu.hltcoe.parse.IlpViterbiParserWithDeltas;
import edu.jhu.hltcoe.parse.IlpViterbiSentenceParser;
import edu.jhu.hltcoe.parse.InitializedIlpViterbiParserWithDeltas;
import edu.jhu.hltcoe.parse.ViterbiParser;
import edu.jhu.hltcoe.util.Command;

/**
 * TODO: Consider switching to annotations
 * http://docs.oracle.com/javase/tutorial/java/javaOO/annotations.html
 * http://tutorials.jenkov.com/java-reflection/annotations.html
 * 
 * @author mgormley
 * 
 */
public class TrainerFactory {

    private static ViterbiParser evalParser = null;

    public static void addOptions(Options options) {
        options.addOption("a", "algorithm", true, "Inference algorithm");
        options.addOption("i", "iterations", true, "Number of iterations");
        options.addOption("i", "convergenceRatio", true, "Convergence ratio");
        options.addOption("nr", "numRestarts", true, "Number of random restarts");
        options.addOption("m", "model", true, "Model");
        options.addOption("p", "parser", true, "Parser");
        options.addOption("d", "initWeights", true, "Method for initializing the weights [uniform, random, supervised]");
        options.addOption("d", "deltaGenerator", true, "Delta generator");
        options.addOption("in", "interval", true, "Only for fixed-interval delta generator");
        options.addOption("fa", "factor", true, "Only for factor delta generator");
        options.addOption("nps", "numPerSide", true, "For symmetric delta generators");
        options.addOption("f", "formulation", true, "ILP formulation for parsing");
        options.addOption("l", "lambda", true, "Value for add-lambda smoothing");
        options.addOption("t", "threads", true, "Number of threads for parallel impl");
        options.addOption("ilp", "ilpSolver", true, "The ILP solver to use " + IlpSolverId.getIdList());
        options.addOption("ilpwmm", "ilpWorkMemMegs", true, "The working memory allotted for the ILP solver in megabytes");
        options.addOption("e", "epsilon", true, "Suboptimality convergence criterion for branch-and-bound");
        options.addOption("vse", "varSelection", true, "Variable selection strategy for branching [full,regret,rand-uniform,rand-weighted]");
        options.addOption("vsp", "varSplit", true, "Variable splitting strategy for branching [half-prob, half-logprob]");
        options.addOption("no", "nodeOrder", true, "Strategy for node selection [bfs, dfs]");
        options.addOption("rx", "relaxation", true, "Relaxation [dw,dw-res,rlt]");
        options.addOption("rx", "maxSimplexIterations", true, "(D-W only) The maximum number of simplex iterations");
        options.addOption("rx", "maxDwIterations", true, "(D-W only) The maximum number of dantzig-wolfe algorithm iterations");
        options.addOption("rx", "maxSetSizeToConstrain", true, "(D-W only) The maximum size of sets to contrain to be <= 1.0");
        options.addOption("rx", "maxCutRounds", true, "(D-W only) The maximum number of rounds to add cuts");
        options.addOption("rx", "minSumForCuts", true, "(D-W only) The minimum threshold at which to stop adding cuts");
        options.addOption("dwt", "dwTempDir", true, "(D-W only) For testing only. The temporary directory to which CPLEX files should be written");
        options.addOption("op", "offsetProb", true, "How much to offset the bounds in probability space from the initial bounds point");
        options.addOption("op", "probOfSkipCm", true, "The probability of not bounding a particular variable");
        options.addOption("op", "timeoutSeconds", true, "The timeout in seconds for training run");
        options.addOption("op", "bnbTimeoutSeconds", true, "[Viterbi-B&B only] The timeout in seconds for branch-and-bound");
        options.addOption("df", "disableFathoming", true, "Disables fathoming in branch-and-bound");
        options.addOption("eo", "envelopeOnly", true, "Whether to use only the convex/concave envelope for the RLT relaxation");
        options.addOption("eo", "rltFilter", true, "RLT filter type [obj-var, prop]");
        options.addOption("eo", "rltInitProp", true, "(prop only) Proportion of initial rows to accept.");
        options.addOption("eo", "rltCutProp", true, "(prop only) Proportion of cut rows to accept.");
        options.addOption("eo", "rltNames", true, "Whether to set RLT variable/constraint names.");
    }

    public static Object getTrainer(CommandLine cmd, DepTreebank trainTreebank, DmvModel trueModel) throws ParseException {

        final String algorithm = Command.getOptionValue(cmd, "algorithm", "viterbi");
        final int iterations = Command.getOptionValue(cmd, "iterations", 10);
        final double convergenceRatio = Command.getOptionValue(cmd, "convergenceRatio", 0.99999);
        final int numRestarts = Command.getOptionValue(cmd, "numRestarts", 0);
        final String modelName = Command.getOptionValue(cmd, "model", "dmv");
        final String parserName = Command.getOptionValue(cmd, "parser", "ilp-sentence");
        final String initWeights = Command.getOptionValue(cmd, "initWeights", "uniform");
        final String deltaGenerator = Command.getOptionValue(cmd, "deltaGenerator", "fixed");
        final double interval = Command.getOptionValue(cmd, "interval", 0.01);
        final double factor = Command.getOptionValue(cmd, "factor", 1.1);
        final int numPerSide = Command.getOptionValue(cmd, "numPerSide", 2);
        final IlpFormulation formulation = getOptionValue(cmd, "formulation", IlpFormulation.DP_PROJ);
        final double lambda = Command.getOptionValue(cmd, "lambda", 0.1);
        final int numThreads = Command.getOptionValue(cmd, "threads", 2);
        final String ilpSolver = Command.getOptionValue(cmd, "ilpSolver", "cplex");
        final double ilpWorkMemMegs = Command.getOptionValue(cmd, "ilpWorkMemMegs", 512.0);
        final double epsilon = Command.getOptionValue(cmd, "epsilon", 0.1);
        final String varSelection = Command.getOptionValue(cmd, "varSelection", "regret");
        final String varSplit = Command.getOptionValue(cmd, "varSplit", "half-prob");
        final String nodeOrder = Command.getOptionValue(cmd, "nodeOrder", "bfs");
        final String relaxation = Command.getOptionValue(cmd, "relaxation", "dw");
        final int maxSimplexIterations = Command.getOptionValue(cmd, "maxSimplexIterations", 2100000000);
        final int maxDwIterations = Command.getOptionValue(cmd, "maxDwIterations", 1000);
        final int maxSetSizeToConstrain = Command.getOptionValue(cmd, "maxSetSizeToConstrain", 2);
        final int maxCutRounds = Command.getOptionValue(cmd, "maxCutRounds", 100);
        final double minSumForCuts = Command.getOptionValue(cmd, "minSumForCuts", 1.01);
        final String dwTempDir = Command.getOptionValue(cmd, "dwTempDir", "");
        final double offsetProb = Command.getOptionValue(cmd, "offsetProb", 1.0);
        final double probOfSkipCm = Command.getOptionValue(cmd, "probOfSkipCm", 0.0);
        final double timeoutSeconds = Command.getOptionValue(cmd, "timeoutSeconds", Double.POSITIVE_INFINITY);
        final double bnbTimeoutSeconds = Command.getOptionValue(cmd, "bnbTimeoutSeconds", Double.POSITIVE_INFINITY);
        final boolean disableFathoming = Command.getOptionValue(cmd, "disableFathoming", false);
        final boolean envelopeOnly = Command.getOptionValue(cmd, "envelopeOnly", true);
        final String rltFilter = Command.getOptionValue(cmd, "rltFilter", "obj-var");
        final double rltInitProp = Command.getOptionValue(cmd, "rltInitProp", 0.1);
        final double rltCutProp = Command.getOptionValue(cmd, "rltCutProp", 0.1);
        final boolean rltNames = Command.getOptionValue(cmd, "rltNames", false);
        
        if (!modelName.equals("dmv")) {
            throw new ParseException("Model not supported: " + modelName);
        }

        double simplexTimeout = algorithm.equals("viterbi-bnb") ? bnbTimeoutSeconds : timeoutSeconds;
        CplexPrm cplexPrm = new CplexPrm(ilpWorkMemMegs, numThreads, maxSimplexIterations, simplexTimeout);

        LpStoBuilderPrm stoPrm = new LpStoBuilderPrm();
        stoPrm.initCutCountComp = new CutCountComputer();
        stoPrm.maxSetSizeToConstrain = maxSetSizeToConstrain;
        stoPrm.minSumForCuts = minSumForCuts;
        
        DmvRelaxation relax = null;
        if (cmd.hasOption("relaxOnly") || algorithm.equals("bnb") || algorithm.equals("viterbi-bnb")) {
            File dwTemp = dwTempDir.equals("") ? null : new File(dwTempDir);
            if (relaxation.equals("dw")) {
                DmvDwRelaxPrm dwPrm = new DmvDwRelaxPrm();
                dwPrm.tempDir = dwTemp;
                dwPrm.maxCutRounds = maxCutRounds;
                dwPrm.cplexPrm = cplexPrm;
                dwPrm.maxDwIterations = maxDwIterations;
                dwPrm.stoPrm = stoPrm;
                relax = new DmvDantzigWolfeRelaxation(dwPrm);
            } else if (relaxation.equals("dw-res")) {
                ResDmvDwRelaxPrm dwPrm = new ResDmvDwRelaxPrm();
                dwPrm.tempDir = dwTemp;
                dwPrm.maxCutRounds = maxCutRounds;
                dwPrm.cplexPrm = cplexPrm;
                dwPrm.maxDwIterations = maxDwIterations;
                relax = new ResDmvDantzigWolfeRelaxation(dwPrm);
            } else if (relaxation.equals("rlt")) {
                RltPrm rltPrm = new RltPrm();
                rltPrm.nameRltVarsAndCons = false;
                rltPrm.envelopeOnly = envelopeOnly;
                rltPrm.nameRltVarsAndCons = rltNames;    
                
                DmvRltRelaxPrm rrPrm = new DmvRltRelaxPrm();
                rrPrm.tempDir = dwTemp;
                rrPrm.maxCutRounds = maxCutRounds;
                rrPrm.cplexPrm = cplexPrm;
                rrPrm.rltPrm = rltPrm;
                rrPrm.stoPrm = stoPrm;
                
                rrPrm.objVarFilter = false;
                if (rltFilter.equals("obj-var")) {
                    rrPrm.objVarFilter = true;
                    rltPrm.factorFilter = null;
                    rltPrm.rowFilter = null;
                } else if (rltFilter.equals("prop")) {
                    rltPrm.rowFilter = new RandPropRltRowFilter(rltInitProp, rltCutProp);
                } else {
                    throw new ParseException("RLT filter type not supported: " + rltFilter);
                }
                relax = new DmvRltRelaxation(rrPrm);
            } else {
                throw new ParseException("Relaxation not supported: " + relaxation);
            }
            if (cmd.hasOption("relaxOnly")) {
                return relax;
            }
        }

        evalParser = new DmvCkyParser();
        DependencyParserEvaluator parserEvaluator = new DependencyParserEvaluator(evalParser, trainTreebank, "train"); 
        
{'fraction_non_alphanumeric': 0.11434023190958462, 'fraction_numerical': 0.004036400998091883, 'mean_word_length': 3.662617621899059, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6637081', 'n_tokens_mistral': 9062, 'n_tokens_neox': 8637, 'n_words': 1738}