<commit_before>package com.cloudera.data;

import com.cloudera.data.partition.HashFieldPartitioner;
import com.cloudera.data.partition.IdentityFieldPartitioner;
import com.cloudera.data.partition.PartitionFunctions;
import org.apache.commons.jexl2.Expression;
import org.apache.commons.jexl2.JexlEngine;

import com.google.common.base.Objects;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class PartitionExpression {

  private JexlEngine engine;
  private Expression expression;
  private boolean isStrict;

  public PartitionExpression(String expression, boolean isStrict) {
    this.engine = new JexlEngine();
    Map<String, Object> fns = new HashMap<String, Object>();
    fns.put(null, PartitionFunctions.class);
    this.engine.setFunctions(fns);
    this.engine.setStrict(true);
    this.engine.setSilent(false);
    this.engine.setCache(10);
    this.expression = engine.createExpression(expression);
    this.isStrict = isStrict;
  }

  public PartitionStrategy evaluate() {
    Object object = expression.evaluate(null);
    if (object instanceof FieldPartitioner) {
      return new PartitionStrategy((FieldPartitioner) object);
    } else if (object instanceof FieldPartitioner[]) {
      return new PartitionStrategy((FieldPartitioner[]) object);
    } else {
      throw new IllegalArgumentException("Partition expression did not produce FieldPartitioner result (or array) for value:" + object);
    }
  }

  /**
   * Convert a PartitionStrategy into a serialized expression. This can be used to set a PartitionStrategy
   * in an Avro property if the PartitionStrategy is passed as an object.
   */
  public static String toExpression(PartitionStrategy partitionStrategy) {
    List<FieldPartitioner> fieldPartitioners = partitionStrategy.getFieldPartitioners();
    if (fieldPartitioners.size() == 1) {
      return toExpression(fieldPartitioners.get(0));
    }
    StringBuilder sb = new StringBuilder();
    sb.append("[");
    for (FieldPartitioner fieldPartitioner : fieldPartitioners) {
      if (sb.length() > 1) {
        sb.append(", ");
      }
      sb.append(toExpression(fieldPartitioner));
    }
    sb.append("]");
    return sb.toString();
  }

  private static String toExpression(FieldPartitioner fieldPartitioner) {
    // TODO: add other strategies
    if (fieldPartitioner instanceof HashFieldPartitioner) {
      return String.format("hash(\"%s\", %s)", fieldPartitioner.getName(), fieldPartitioner.getCardinality());
    } else if (fieldPartitioner instanceof IdentityFieldPartitioner) {
      return String.format("identity(\"%s\", %s)", fieldPartitioner.getName(), fieldPartitioner.getCardinality());
    }
    throw new IllegalArgumentException("Unrecognized PartitionStrategy: " + fieldPartitioner);
  }

  @Override
  public String toString() {
    return Objects.toStringHelper(this).add("expression", expression)
        .add("isStrict", isStrict).add("engine", engine).toString();
  }

}<commit_msg>The Eclipse reformatter strikes again.<commit_after>package com.cloudera.data;

import com.cloudera.data.partition.HashFieldPartitioner;
import com.cloudera.data.partition.IdentityFieldPartitioner;
import com.cloudera.data.partition.PartitionFunctions;
import org.apache.commons.jexl2.Expression;
import org.apache.commons.jexl2.JexlEngine;

import com.google.common.base.Objects;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class PartitionExpression {

  private JexlEngine engine;
  private Expression expression;
  private boolean isStrict;

  public PartitionExpression(String expression, boolean isStrict) {
    this.engine = new JexlEngine();
    Map<String, Object> fns = new HashMap<String, Object>();
    fns.put(null, PartitionFunctions.class);
    this.engine.setFunctions(fns);
    this.engine.setStrict(true);
    this.engine.setSilent(false);
    this.engine.setCache(10);
    this.expression = engine.createExpression(expression);
    this.isStrict = isStrict;
  }

  public PartitionStrategy evaluate() {
    Object object = expression.evaluate(null);
    if (object instanceof FieldPartitioner) {
      return new PartitionStrategy((FieldPartitioner) object);
    } else if (object instanceof FieldPartitioner[]) {
      return new PartitionStrategy((FieldPartitioner[]) object);
    } else {
      throw new IllegalArgumentException(
          "Partition expression did not produce FieldPartitioner result (or array) for value:"
              + object);
    }
  }

  /**
   * Convert a PartitionStrategy into a serialized expression. This can be used
   * to set a PartitionStrategy in an Avro property if the PartitionStrategy is
   * passed as an object.
   */
  public static String toExpression(PartitionStrategy partitionStrategy) {
    List<FieldPartitioner> fieldPartitioners = partitionStrategy
        .getFieldPartitioners();
    if (fieldPartitioners.size() == 1) {
      return toExpression(fieldPartitioners.get(0));
    }
    StringBuilder sb = new StringBuilder();
    sb.append("[");
    for (FieldPartitioner fieldPartitioner : fieldPartitioners) {
      if (sb.length() > 1) {
        sb.append(", ");
      }
      sb.append(toExpression(fieldPartitioner));
    }
    sb.append("]");
    return sb.toString();
  }

  private static String toExpression(FieldPartitioner fieldPartitioner) {
    // TODO: add other strategies
    if (fieldPartitioner instanceof HashFieldPartitioner) {
      return String.format("hash(\"%s\", %s)", fieldPartitioner.getName(),
          fieldPartitioner.getCardinality());
    } else if (fieldPartitioner instanceof IdentityFieldPartitioner) {
      return String.format("identity(\"%s\", %s)", fieldPartitioner.getName(),
          fieldPartitioner.getCardinality());
    }
    throw new IllegalArgumentException("Unrecognized PartitionStrategy: "
        + fieldPartitioner);
  }

  @Override
  public String toString() {
    return Objects.toStringHelper(this).add("expression", expression)
        .add("isStrict", isStrict).add("engine", engine).toString();
  }

}
{'fraction_non_alphanumeric': 0.10868847043277172, 'fraction_numerical': 0.0023125206475057814, 'mean_word_length': 4.1400679117147705, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '11267', 'n_tokens_mistral': 1669, 'n_tokens_neox': 1656, 'n_words': 394}