<commit_before>package com.microsoft.tang.formats;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

//import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.Parser;

import com.microsoft.tang.ConfigurationBuilder;
import com.microsoft.tang.annotations.Name;
import com.microsoft.tang.exceptions.BindException;
import com.microsoft.tang.exceptions.NameResolutionException;
import com.microsoft.tang.types.NamedParameterNode;
import com.microsoft.tang.types.Node;
import com.microsoft.tang.util.MonotonicMap;
import com.microsoft.tang.util.ReflectionUtilities;

public class CommandLine {
  private final ConfigurationBuilder conf;
  private final Map<String,String> shortNames = new MonotonicMap<>();

  public CommandLine(ConfigurationBuilder conf) {
    this.conf = conf;
  }
  public void registerShortNameOfClass(String s) throws BindException {
    final Node n;
    try {
      n = conf.getClassHierarchy().getNode(s);
    } catch(NameResolutionException e) {
      throw new BindException("Problem loading class " + s, e);
    }
    if(n instanceof NamedParameterNode) {
      NamedParameterNode<?> np = (NamedParameterNode<?>)n;
      String shortName = np.getShortName();
      String longName = np.getFullName();
      shortNames.put(shortName, longName);
    } else {
      throw new BindException("Can't register short name for non-NamedParameterNode: " + n);
    }
  }
  public void registerShortNameOfClass(Class<? extends Name<?>> c) throws BindException {
    registerShortNameOfClass(ReflectionUtilities.getFullName(c));
  }
  /**
   * @param option
   * @param cb
   */
  @SuppressWarnings("static-access")
  private Options getCommandLineOptions() {
    Options opts = new Options();
    // Collection<NamedParameterNode<?>> namedParameters = conf.getNamespace()
    // .getNamedParameterNodes();
    // for (NamedParameterNode<?> param : namedParameters) {
    // String shortName = param.getShortName();
    for (String shortName : shortNames.keySet()) {
      String longName = shortNames.get(shortName);
      try {
        opts.addOption(OptionBuilder
            .withArgName(conf.classPrettyDefaultString(longName)).hasArg()
            .withDescription(conf.classPrettyDescriptionString(longName))
            .create(shortName));
      } catch (BindException e) {
        throw new IllegalStateException(
            "Could not process " + shortName + " which is the short name of " + longName, e);
      }
    }
    for (Option o : applicationOptions.keySet()) {
      opts.addOption(o);
    }
    return opts;
  }

  public interface CommandLineCallback {
    public void process(Option option);
  }

  Map<Option, CommandLineCallback> applicationOptions = new HashMap<>();

  public void addCommandLineOption(Option option, CommandLineCallback cb) {
    // TODO: Check for conflicting options.
    applicationOptions.put(option, cb);
  }

  /**
   * @return true if the command line parsing succeeded, false (or exception)
   *         otherwise.
   * @param args
   * @throws IOException
   * @throws NumberFormatException
   * @throws ParseException
   */
  public <T> boolean processCommandLine(String[] args) throws IOException,
      BindException {
    Options o = getCommandLineOptions();
    Option helpFlag = new Option("?", "help");
    o.addOption(helpFlag);
    Parser g = new GnuParser();
    org.apache.commons.cli.CommandLine cl;
    try {
      cl = g.parse(o, args);
    } catch (ParseException e) {
      throw new IOException("Could not parse config file", e);
    }
    if (cl.hasOption("?")) {
      HelpFormatter help = new HelpFormatter();
      help.printHelp("reef", o);
      return false;
    }
    for (Option option : cl.getOptions()) {
      String shortName = option.getOpt();
      String value = option.getValue();

      if (applicationOptions.containsKey(option)) {
        applicationOptions.get(option).process(option);
      } else {
        try {
          conf.bind(shortNames.get(shortName), value);
        } catch (BindException e) {
          throw new BindException("Could not bind shortName " + shortName + " to value " + value, e);
        }
      }
    }
    return true;
  }

}
<commit_msg>processCommandLine now takes an optional list of argument classes.<commit_after>package com.microsoft.tang.formats;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

//import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.Parser;

import com.microsoft.tang.ConfigurationBuilder;
import com.microsoft.tang.annotations.Name;
import com.microsoft.tang.exceptions.BindException;
import com.microsoft.tang.exceptions.NameResolutionException;
import com.microsoft.tang.types.NamedParameterNode;
import com.microsoft.tang.types.Node;
import com.microsoft.tang.util.MonotonicMap;
import com.microsoft.tang.util.ReflectionUtilities;

public class CommandLine {
  private final ConfigurationBuilder conf;
  private final Map<String,String> shortNames = new MonotonicMap<>();

  public CommandLine(ConfigurationBuilder conf) {
    this.conf = conf;
  }
  public void registerShortNameOfClass(String s) throws BindException {
    final Node n;
    try {
      n = conf.getClassHierarchy().getNode(s);
    } catch(NameResolutionException e) {
      throw new BindException("Problem loading class " + s, e);
    }
    if(n instanceof NamedParameterNode) {
      NamedParameterNode<?> np = (NamedParameterNode<?>)n;
      String shortName = np.getShortName();
      String longName = np.getFullName();
      shortNames.put(shortName, longName);
    } else {
      throw new BindException("Can't register short name for non-NamedParameterNode: " + n);
    }
  }
  public void registerShortNameOfClass(Class<? extends Name<?>> c) throws BindException {
    registerShortNameOfClass(ReflectionUtilities.getFullName(c));
  }
  /**
   * @param option
   * @param cb
   */
  @SuppressWarnings("static-access")
  private Options getCommandLineOptions() {
    Options opts = new Options();
    // Collection<NamedParameterNode<?>> namedParameters = conf.getNamespace()
    // .getNamedParameterNodes();
    // for (NamedParameterNode<?> param : namedParameters) {
    // String shortName = param.getShortName();
    for (String shortName : shortNames.keySet()) {
      String longName = shortNames.get(shortName);
      try {
        opts.addOption(OptionBuilder
            .withArgName(conf.classPrettyDefaultString(longName)).hasArg()
            .withDescription(conf.classPrettyDescriptionString(longName))
            .create(shortName));
      } catch (BindException e) {
        throw new IllegalStateException(
            "Could not process " + shortName + " which is the short name of " + longName, e);
      }
    }
    for (Option o : applicationOptions.keySet()) {
      opts.addOption(o);
    }
    return opts;
  }

  public interface CommandLineCallback {
    public void process(Option option);
  }

  Map<Option, CommandLineCallback> applicationOptions = new HashMap<>();

  public void addCommandLineOption(Option option, CommandLineCallback cb) {
    // TODO: Check for conflicting options.
    applicationOptions.put(option, cb);
  }

  /**
   * @return true if the command line parsing succeeded, false (or exception)
   *         otherwise.
   * @param args
   * @throws IOException
   * @throws NumberFormatException
   * @throws ParseException
   */
  public <T> boolean processCommandLine(String[] args, @SuppressWarnings("unchecked") Class<? extends Name<?>>...argClasses) throws IOException,
      BindException {
    for(Class<? extends Name<?>> c : argClasses) {
      registerShortNameOfClass(c);
    }
    Options o = getCommandLineOptions();
    Option helpFlag = new Option("?", "help");
    o.addOption(helpFlag);
    Parser g = new GnuParser();
    org.apache.commons.cli.CommandLine cl;
    try {
      cl = g.parse(o, args);
    } catch (ParseException e) {
      throw new IOException("Could not parse config file", e);
    }
    if (cl.hasOption("?")) {
      HelpFormatter help = new HelpFormatter();
      help.printHelp("reef", o);
      return false;
    }
    for (Option option : cl.getOptions()) {
      String shortName = option.getOpt();
      String value = option.getValue();

      if (applicationOptions.containsKey(option)) {
        applicationOptions.get(option).process(option);
      } else {
        try {
          conf.bind(shortNames.get(shortName), value);
        } catch (BindException e) {
          throw new BindException("Could not bind shortName " + shortName + " to value " + value, e);
        }
      }
    }
    return true;
  }

}

{'fraction_non_alphanumeric': 0.11316220644456582, 'fraction_numerical': 0.0, 'mean_word_length': 3.4231884057971014, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1462128', 'n_tokens_mistral': 2607, 'n_tokens_neox': 2576, 'n_words': 614}