<commit_before>package com.bbn.bue.common.parameters;

import com.bbn.bue.common.StringUtils;
import com.bbn.bue.common.converters.StrictStringToBoolean;
import com.bbn.bue.common.converters.StringConverter;
import com.bbn.bue.common.converters.StringToDouble;
import com.bbn.bue.common.converters.StringToEnum;
import com.bbn.bue.common.converters.StringToFile;
import com.bbn.bue.common.converters.StringToInteger;
import com.bbn.bue.common.converters.StringToOSFile;
import com.bbn.bue.common.converters.StringToStringList;
import com.bbn.bue.common.converters.StringToStringSet;
import com.bbn.bue.common.converters.StringToSymbolList;
import com.bbn.bue.common.converters.StringToSymbolSet;
import com.bbn.bue.common.parameters.exceptions.InvalidEnumeratedPropertyException;
import com.bbn.bue.common.parameters.exceptions.MissingRequiredParameter;
import com.bbn.bue.common.parameters.exceptions.ParameterConversionException;
import com.bbn.bue.common.parameters.exceptions.ParameterException;
import com.bbn.bue.common.parameters.exceptions.ParameterValidationException;
import com.bbn.bue.common.parameters.serifstyle.SerifStyleParameterFileLoader;
import com.bbn.bue.common.symbols.Symbol;
import com.bbn.bue.common.validators.AlwaysValid;
import com.bbn.bue.common.validators.And;
import com.bbn.bue.common.validators.FileExists;
import com.bbn.bue.common.validators.IsDirectory;
import com.bbn.bue.common.validators.IsFile;
import com.bbn.bue.common.validators.IsInRange;
import com.bbn.bue.common.validators.IsNonNegative;
import com.bbn.bue.common.validators.IsPositive;
import com.bbn.bue.common.validators.ValidationException;
import com.bbn.bue.common.validators.Validator;

import com.google.common.annotations.Beta;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Range;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.reflect.InvocationTargetException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;


/**
 * Represents a set of parameters passed into a program.  The parameters are assumed to originate as
 * key-value pairs of <code>String</code>s, which can then be accessed in various validated ways.
 * This class is immutable. Keys will never be null or empty. Values will never be null.
 *
 * For all methods to get parameters, looking up a missing parameter throws an unchecked {@link
 * MissingRequiredParameter} exception.
 *
 * @author rgabbard
 */
@Beta
public final class Parameters {

  public static final String DO_OS_CONVERSION_PARAM = "os_filepath_conversion";

  /**
   * Constructs a Parameters object from a <code>Map</code>.  The Map may contain neither null keys,
   * empty keys, or null values.
   *
   * @deprecated Prefer fromMap()
   */
  @Deprecated
  public Parameters(final Map<String, String> params) {
    this(params, ImmutableList.<String>of());
  }

  private Parameters(final Map<String, String> params, final List<String> namespace) {
    this.namespace = ImmutableList.copyOf(namespace);
    this.params = ImmutableMap.copyOf(params);
    for (final Map.Entry<String, String> param : params.entrySet()) {
      checkNotNull(param.getKey());
      checkNotNull(param.getValue());
      checkArgument(!param.getKey().isEmpty());
    }
  }

  /**
   * Creates a new set of parameters with only those parameters in the specified namespace (that is,
   * prefixed by "namespace.". The namespace prefix and period will be removed from parameter names
   * in the new {@code Parameters}.
   */
  public Parameters copyNamespace(final String requestedNamespace) {
    checkArgument(!requestedNamespace.isEmpty());
    final ImmutableMap.Builder<String, String> ret = ImmutableMap.builder();
    final String dottedNamespace = requestedNamespace + ".";
    for (final Map.Entry<String, String> param : params.entrySet()) {
      if (param.getKey().startsWith(dottedNamespace)) {
        ret.put(param.getKey().substring(dottedNamespace.length()), param.getValue());
      }
    }
    final List<String> newNamespace = Lists.newArrayList();
    newNamespace.addAll(namespace);
    newNamespace.add(requestedNamespace);
    return new Parameters(ret.build(), newNamespace);
  }

  /**
   * If the specified namespace is present, return a copy of that namespace as a parameter set.
   * Otherwise, return a copy of this parameter set.
   */
  public Parameters copyNamespaceIfPresent(final String requestedNamespace) {
    if (isNamespacePresent(requestedNamespace)) {
      return copyNamespace(requestedNamespace);
    } else {
      return copy();
    }
  }

  /**
   * Returns if any parameter in this parameter set begins the the specified string, followed by a
   * dot. The argument may not be empty.
   */
  public boolean isNamespacePresent(final String requestedNamespace) {
    checkArgument(requestedNamespace.length() > 0);
    final String probe = requestedNamespace + ".";
    return Iterables.any(params.keySet(), StringUtils.startsWith(probe));
  }

  /**
   * Creates a copy of this parameter set.
   */
  public Parameters copy() {
    return new Parameters(params, namespace);
  }

  public String dump() {
    return dump(true, true);
  }

  public String dumpWithoutNamespacePrefix() {
    return dump(true, false);
  }

  public String dump(final boolean printDateTime) {
    return dump(printDateTime, true);
  }

  /**
   * Dumps the parameters object as colon-separated key-value pairs. If {@code printDateTime} is
   * true, will put a #-style comment with the current date and time at the top. If
   * includeNamespacePrefix is true, will prefix its parameter with its full namespace instead of
   * writing all keys relative to the current namespace.
   */
  public String dump(final boolean printDateTime, final boolean includeNamespacePrefix) {
    final StringWriter sOut = new StringWriter();
    final PrintWriter out = new PrintWriter(sOut);

    if (printDateTime) {
      // output a timestamp comment
      final SimpleDateFormat timeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
      out.format("#%s\n", timeFormat.format(new Date()));
    }
    for (final Map.Entry<String, String> param : params.entrySet()) {
      final String key;
      if (includeNamespacePrefix) {
        key = fullString(param.getKey());
      } else {
        key = param.getKey();
      }
      out.format("%s: %s\n", key, param.getValue());
    }

    out.close();
    return sOut.toString();<commit_msg>Sort by key before dumping param key-value pairs.<commit_after>package com.bbn.bue.common.parameters;

import com.bbn.bue.common.StringUtils;
import com.bbn.bue.common.converters.StrictStringToBoolean;
import com.bbn.bue.common.converters.StringConverter;
import com.bbn.bue.common.converters.StringToDouble;
import com.bbn.bue.common.converters.StringToEnum;
import com.bbn.bue.common.converters.StringToFile;
import com.bbn.bue.common.converters.StringToInteger;
import com.bbn.bue.common.converters.StringToOSFile;
import com.bbn.bue.common.converters.StringToStringList;
import com.bbn.bue.common.converters.StringToStringSet;
import com.bbn.bue.common.converters.StringToSymbolList;
import com.bbn.bue.common.converters.StringToSymbolSet;
import com.bbn.bue.common.parameters.exceptions.InvalidEnumeratedPropertyException;
import com.bbn.bue.common.parameters.exceptions.MissingRequiredParameter;
import com.bbn.bue.common.parameters.exceptions.ParameterConversionException;
import com.bbn.bue.common.parameters.exceptions.ParameterException;
import com.bbn.bue.common.parameters.exceptions.ParameterValidationException;
import com.bbn.bue.common.parameters.serifstyle.SerifStyleParameterFileLoader;
import com.bbn.bue.common.symbols.Symbol;
import com.bbn.bue.common.validators.AlwaysValid;
import com.bbn.bue.common.validators.And;
import com.bbn.bue.common.validators.FileExists;
import com.bbn.bue.common.validators.IsDirectory;
import com.bbn.bue.common.validators.IsFile;
import com.bbn.bue.common.validators.IsInRange;
import com.bbn.bue.common.validators.IsNonNegative;
import com.bbn.bue.common.validators.IsPositive;
import com.bbn.bue.common.validators.ValidationException;
import com.bbn.bue.common.validators.Validator;

import com.google.common.annotations.Beta;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Range;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.reflect.InvocationTargetException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;


/**
 * Represents a set of parameters passed into a program.  The parameters are assumed to originate as
 * key-value pairs of <code>String</code>s, which can then be accessed in various validated ways.
 * This class is immutable. Keys will never be null or empty. Values will never be null.
 *
 * For all methods to get parameters, looking up a missing parameter throws an unchecked {@link
 * MissingRequiredParameter} exception.
 *
 * @author rgabbard
 */
@Beta
public final class Parameters {

  public static final String DO_OS_CONVERSION_PARAM = "os_filepath_conversion";

  /**
   * Constructs a Parameters object from a <code>Map</code>.  The Map may contain neither null keys,
   * empty keys, or null values.
   *
   * @deprecated Prefer fromMap()
   */
  @Deprecated
  public Parameters(final Map<String, String> params) {
    this(params, ImmutableList.<String>of());
  }

  private Parameters(final Map<String, String> params, final List<String> namespace) {
    this.namespace = ImmutableList.copyOf(namespace);
    this.params = ImmutableMap.copyOf(params);
    for (final Map.Entry<String, String> param : params.entrySet()) {
      checkNotNull(param.getKey());
      checkNotNull(param.getValue());
      checkArgument(!param.getKey().isEmpty());
    }
  }

  /**
   * Creates a new set of parameters with only those parameters in the specified namespace (that is,
   * prefixed by "namespace.". The namespace prefix and period will be removed from parameter names
   * in the new {@code Parameters}.
   */
  public Parameters copyNamespace(final String requestedNamespace) {
    checkArgument(!requestedNamespace.isEmpty());
    final ImmutableMap.Builder<String, String> ret = ImmutableMap.builder();
    final String dottedNamespace = requestedNamespace + ".";
    for (final Map.Entry<String, String> param : params.entrySet()) {
      if (param.getKey().startsWith(dottedNamespace)) {
        ret.put(param.getKey().substring(dottedNamespace.length()), param.getValue());
      }
    }
    final List<String> newNamespace = Lists.newArrayList();
    newNamespace.addAll(namespace);
    newNamespace.add(requestedNamespace);
    return new Parameters(ret.build(), newNamespace);
  }

  /**
   * If the specified namespace is present, return a copy of that namespace as a parameter set.
   * Otherwise, return a copy of this parameter set.
   */
  public Parameters copyNamespaceIfPresent(final String requestedNamespace) {
    if (isNamespacePresent(requestedNamespace)) {
      return copyNamespace(requestedNamespace);
    } else {
      return copy();
    }
  }

  /**
   * Returns if any parameter in this parameter set begins the the specified string, followed by a
   * dot. The argument may not be empty.
   */
  public boolean isNamespacePresent(final String requestedNamespace) {
    checkArgument(requestedNamespace.length() > 0);
    final String probe = requestedNamespace + ".";
    return Iterables.any(params.keySet(), StringUtils.startsWith(probe));
  }

  /**
   * Creates a copy of this parameter set.
   */
  public Parameters copy() {
    return new Parameters(params, namespace);
  }

  public String dump() {
    return dump(true, true);
  }

  public String dumpWithoutNamespacePrefix() {
    return dump(true, false);
  }

  public String dump(final boolean printDateTime) {
    return dump(printDateTime, true);
  }

  /**
   * Dumps the parameters object as colon-separated key-value pairs. If {@code printDateTime} is
   * true, will put a #-style comment with the current date and time at the top. If
   * includeNamespacePrefix is true, will prefix its parameter with its full namespace instead of
   * writing all keys relative to the current namespace.
   */
  public String dump(final boolean printDateTime, final boolean includeNamespacePrefix) {
    final StringWriter sOut = new StringWriter();
    final PrintWriter out = new PrintWriter(sOut);

    if (printDateTime) {
      // output a timestamp comment
      final SimpleDateFormat timeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
      out.format("#%s\n", timeFormat.format(new Date()));
    }
    List<String> keys = new ArrayList<String>(params.keySet());
    Collections.sort(keys);
    for (final String rawKey : keys) {
      final String key;
      if (includeNamespacePrefix) {
        key = fullString(rawKey);
      } else {
        key = rawKey;
      }
      out.format("%s: %s\n", key, params.get(rawKey));
    }

    out.close();
    return sOut.toString();
{'fraction_non_alphanumeric': 0.10507530873010208, 'fraction_numerical': 0.0001427653651224213, 'mean_word_length': 4.308829101932551, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2499783', 'n_tokens_mistral': 4085, 'n_tokens_neox': 4033, 'n_words': 1038}