<commit_before>package com.jayantkrish.jklol.cli;

import java.util.List;

import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.jayantkrish.jklol.evaluation.Example;
import com.jayantkrish.jklol.inference.JunctionTree;
import com.jayantkrish.jklol.models.DiscreteVariable;
import com.jayantkrish.jklol.models.ObjectVariable;
import com.jayantkrish.jklol.models.TableFactor;
import com.jayantkrish.jklol.models.VariableNumMap;
import com.jayantkrish.jklol.models.dynamic.DynamicFactorGraph;
import com.jayantkrish.jklol.models.loglinear.ConditionalLogLinearFactor;
import com.jayantkrish.jklol.models.parametric.ParametricFactorGraph;
import com.jayantkrish.jklol.models.parametric.ParametricFactorGraphBuilder;
import com.jayantkrish.jklol.models.parametric.SufficientStatistics;
import com.jayantkrish.jklol.tensor.Tensor;
import com.jayantkrish.jklol.training.MaxMarginOracle;
import com.jayantkrish.jklol.training.MaxMarginOracle.HammingCost;
import com.jayantkrish.jklol.training.OracleAdapter;
import com.jayantkrish.jklol.training.StochasticGradientTrainer;
import com.jayantkrish.jklol.util.Assignment;
import com.jayantkrish.jklol.util.IoUtils;

/**
 * Command line program for training a linear classifier.
 * 
 * @author jayantk
 */
public class TrainLinearClassifier extends AbstractCli {
  
  private OptionSpec<String> featureVectorFile;
  private OptionSpec<String> labelFile;
  private OptionSpec<String> modelOutput;
  private OptionSpec<String> delimiterOption;
  
  private static final String INPUT_VAR_NAME = "x";
  private static final String OUTPUT_VAR_NAME = "y";
  
  public TrainLinearClassifier() {
    super(CommonOptions.STOCHASTIC_GRADIENT, CommonOptions.MAP_REDUCE);
  }

  @Override
  public void initializeOptions(OptionParser parser) {
    // Required arguments.
    featureVectorFile = parser.accepts("features").withRequiredArg().ofType(String.class).required();
    labelFile = parser.accepts("labels").withRequiredArg().ofType(String.class).required();
    modelOutput = parser.accepts("output").withRequiredArg().ofType(String.class).required();
    // Optional options
    delimiterOption = parser.accepts("delimiter").withRequiredArg().ofType(String.class)
        .defaultsTo(",");
  }

  @Override
  public void run(OptionSet options) {
    String delimiter = options.valueOf(delimiterOption);
    // The first column of both files is the set of example IDs.
    List<String> exampleIds = IoUtils.readUniqueColumnValuesFromDelimitedFile(
        options.valueOf(featureVectorFile), 0, delimiter);
    List<String> featureNames = IoUtils.readUniqueColumnValuesFromDelimitedFile(
        options.valueOf(featureVectorFile), 1, delimiter);
    List<String> labelNames = IoUtils.readUniqueColumnValuesFromDelimitedFile(<commit_msg>basically nothing in this commit<commit_after>package com.jayantkrish.jklol.cli;

import java.util.List;

import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.jayantkrish.jklol.evaluation.Example;
import com.jayantkrish.jklol.inference.JunctionTree;
import com.jayantkrish.jklol.models.DiscreteVariable;
import com.jayantkrish.jklol.models.ObjectVariable;
import com.jayantkrish.jklol.models.TableFactor;
import com.jayantkrish.jklol.models.VariableNumMap;
import com.jayantkrish.jklol.models.dynamic.DynamicFactorGraph;
import com.jayantkrish.jklol.models.loglinear.ConditionalLogLinearFactor;
import com.jayantkrish.jklol.models.parametric.ParametricFactorGraph;
import com.jayantkrish.jklol.models.parametric.ParametricFactorGraphBuilder;
import com.jayantkrish.jklol.models.parametric.SufficientStatistics;
import com.jayantkrish.jklol.tensor.Tensor;
import com.jayantkrish.jklol.training.MaxMarginOracle;
import com.jayantkrish.jklol.training.MaxMarginOracle.HammingCost;
import com.jayantkrish.jklol.training.OracleAdapter;
import com.jayantkrish.jklol.training.StochasticGradientTrainer;
import com.jayantkrish.jklol.util.Assignment;
import com.jayantkrish.jklol.util.IoUtils;

/**
 * Command line program for training a linear classifier.
 * 
 * @author jayantk
 */
public class TrainLinearClassifier extends AbstractCli {
  
  private OptionSpec<String> featureVectorFile;
  private OptionSpec<String> labelFile;
  private OptionSpec<String> modelOutput;
  private OptionSpec<String> delimiterOption;
  
  private static final String INPUT_VAR_NAME = "x";
  private static final String OUTPUT_VAR_NAME = "y";
  
  public TrainLinearClassifier() {
    super(CommonOptions.STOCHASTIC_GRADIENT, CommonOptions.MAP_REDUCE);
  }

  @Override
  public void initializeOptions(OptionParser parser) {
    // Required arguments.
    featureVectorFile = parser.accepts("features").withRequiredArg().ofType(String.class).required();
    labelFile = parser.accepts("labels").withRequiredArg().ofType(String.class).required();
    modelOutput = parser.accepts("output").withRequiredArg().ofType(String.class).required();

    // Optional options
    delimiterOption = parser.accepts("delimiter").withRequiredArg().ofType(String.class)
        .defaultsTo(",");
  }

  @Override
  public void run(OptionSet options) {
    String delimiter = options.valueOf(delimiterOption);
    // The first column of both files is the set of example IDs.
    List<String> exampleIds = IoUtils.readUniqueColumnValuesFromDelimitedFile(
        options.valueOf(featureVectorFile), 0, delimiter);
    List<String> featureNames = IoUtils.readUniqueColumnValuesFromDelimitedFile(
        options.valueOf(featureVectorFile), 1, delimiter);
    List<String> labelNames = IoUtils.readUniqueColumnValuesFromDelimitedFile(
{'fraction_non_alphanumeric': 0.09780370624571036, 'fraction_numerical': 0.0006863417982155113, 'mean_word_length': 5.890070921985815, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '133928', 'n_tokens_mistral': 1845, 'n_tokens_neox': 1778, 'n_words': 295}