<commit_before>package com.cloudera.data;

import javax.annotation.Nullable;

import org.apache.avro.Schema;

import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.DataFileStream;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericRecord;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;

/**
 * <p>
 * The structural definition of a {@link Dataset}.
 * </p>
 * <p>
 * Each {@code Dataset} has an associated {@link Schema} and optional
 * {@link PartitionStrategy} defined at the time of creation. Instances of this
 * class are used to hold this information. Users are strongly encouraged to use
 * the inner {@link Builder} to create new instances.
 * </p>
 */
public class DatasetDescriptor {

  private Schema schema;
  private PartitionStrategy partitionStrategy;

  /**
   * Create an instance of this class with the supplied {@link Schema} and
   * optional {@link PartitionStrategy}.
   */
  public DatasetDescriptor(Schema schema,
      @Nullable PartitionStrategy partitionStrategy) {

    this.schema = schema;
    this.partitionStrategy = partitionStrategy;
  }

  public Schema getSchema() {
    return schema;
  }

  @Nullable
  public PartitionStrategy getPartitionStrategy() {
    return partitionStrategy;
  }

  @Override
  public String toString() {
    return Objects.toStringHelper(this).add("schema", schema)
        .add("partitionStrategy", partitionStrategy).toString();
  }

  /**
   * A fluent builder to aid in the construction of {@link DatasetDescriptor}s.
   */
  public static class Builder implements Supplier<DatasetDescriptor> {

    private Schema schema;
    private PartitionStrategy partitionStrategy;

    /**
     * Configure the dataset's schema. Required.
     * 
     * @return An instance of the builder for method chaining.
     */
    public Builder schema(Schema schema) {
      this.schema = schema;
      return this;
    }

    /**
     * Configure the dataset's schema from a {@link File}. Required.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schema(File file) throws IOException {
      this.schema = new Schema.Parser().parse(file);
      return this;
    }

    /**
     * Configure the dataset's schema from an {@link InputStream}. Required.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schema(InputStream in) throws IOException {
      this.schema = new Schema.Parser().parse(in);
      return this;
    }

    /**
     * Configure the dataset's schema from a {@link URL}. Required.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schema(URL url) throws IOException {
      InputStream in = null;
      try {
        in = url.openStream();
        return schema(in);
      } finally {
        if (in != null) {
          in.close();
        }
      }
    }

    /**
     * Configure the dataset's schema from a {@link String}. Required.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schema(String s) throws IOException {
      this.schema = new Schema.Parser().parse(s);
      return this;
    }

    /**
     * Configure the dataset's schema by using the schema from an existing Avro data
     * file.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schemaFromAvroDataFile(File file) throws IOException {
      GenericDatumReader<GenericRecord> datumReader =
          new GenericDatumReader<GenericRecord>();
      DataFileReader<GenericRecord> reader = new DataFileReader<GenericRecord>(file,
          datumReader);
      this.schema = reader.getSchema();
      return this;
    }

    /**
     * Configure the dataset's schema by using the schema from an existing Avro data
     * file.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schemaFromAvroDataFile(InputStream in) throws IOException {
      GenericDatumReader<GenericRecord> datumReader =
          new GenericDatumReader<GenericRecord>();
      DataFileStream<GenericRecord> stream = new DataFileStream<GenericRecord>(in,
          datumReader);
      this.schema = stream.getSchema();
      return this;
    }

    /**
     * Configure the dataset's schema by using the schema from an existing Avro data
     * file.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schemaFromAvroDataFile(URL url) throws IOException {
      InputStream in = null;
      try {
        in = url.openStream();
        return schemaFromAvroDataFile(in);
      } finally {
        if (in != null) {
          in.close();
        }
      }
    }

    /**
     * Configure the dataset's partitioning strategy. Optional.
     * 
     * @return An instance of the builder for method chaining.<commit_msg>Addressed Eric's feedback on f44cb893fc16a3e49e6d5fce7827d419c8a4f2f3 commit.<commit_after>package com.cloudera.data;

import javax.annotation.Nullable;

import com.google.common.io.Closeables;
import org.apache.avro.Schema;

import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.DataFileStream;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericRecord;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;

/**
 * <p>
 * The structural definition of a {@link Dataset}.
 * </p>
 * <p>
 * Each {@code Dataset} has an associated {@link Schema} and optional
 * {@link PartitionStrategy} defined at the time of creation. Instances of this
 * class are used to hold this information. Users are strongly encouraged to use
 * the inner {@link Builder} to create new instances.
 * </p>
 */
public class DatasetDescriptor {

  private Schema schema;
  private PartitionStrategy partitionStrategy;

  /**
   * Create an instance of this class with the supplied {@link Schema} and
   * optional {@link PartitionStrategy}.
   */
  public DatasetDescriptor(Schema schema,
      @Nullable PartitionStrategy partitionStrategy) {

    this.schema = schema;
    this.partitionStrategy = partitionStrategy;
  }

  public Schema getSchema() {
    return schema;
  }

  @Nullable
  public PartitionStrategy getPartitionStrategy() {
    return partitionStrategy;
  }

  @Override
  public String toString() {
    return Objects.toStringHelper(this).add("schema", schema)
        .add("partitionStrategy", partitionStrategy).toString();
  }

  /**
   * A fluent builder to aid in the construction of {@link DatasetDescriptor}s.
   */
  public static class Builder implements Supplier<DatasetDescriptor> {

    private Schema schema;
    private PartitionStrategy partitionStrategy;

    /**
     * Configure the dataset's schema.
     * A schema is required, and may be set using one of the <code>schema</code> or
     * <code>schemaFromAvroDataFile</code> methods.
     * 
     * @return An instance of the builder for method chaining.
     */
    public Builder schema(Schema schema) {
      this.schema = schema;
      return this;
    }

    /**
     * Configure the dataset's schema from a {@link File}.
     * A schema is required, and may be set using one of the <code>schema</code> or
     * <code>schemaFromAvroDataFile</code> methods.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schema(File file) throws IOException {
      this.schema = new Schema.Parser().parse(file);
      return this;
    }

    /**
     * Configure the dataset's schema from an {@link InputStream}. It is the caller's
     * responsibility to close the {@link InputStream}.
     * A schema is required, and may be set using one of the <code>schema</code> or
     * <code>schemaFromAvroDataFile</code> methods.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schema(InputStream in) throws IOException {
      this.schema = new Schema.Parser().parse(in);
      return this;
    }

    /**
     * Configure the dataset's schema from a {@link URL}.
     * A schema is required, and may be set using one of the <code>schema</code> or
     * <code>schemaFromAvroDataFile</code> methods.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schema(URL url) throws IOException {
      InputStream in = null;
      try {
        in = url.openStream();
        return schema(in);
      } finally {
        Closeables.closeQuietly(in);
      }
    }

    /**
     * Configure the dataset's schema from a {@link String}.
     * A schema is required, and may be set using one of the <code>schema</code> or
     * <code>schemaFromAvroDataFile</code> methods.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schema(String s) throws IOException {
      this.schema = new Schema.Parser().parse(s);
      return this;
    }

    /**
     * Configure the dataset's schema by using the schema from an existing Avro data
     * file.
     * A schema is required, and may be set using one of the <code>schema</code> or
     * <code>schemaFromAvroDataFile</code> methods.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schemaFromAvroDataFile(File file) throws IOException {
      GenericDatumReader<GenericRecord> datumReader =
          new GenericDatumReader<GenericRecord>();
      DataFileReader<GenericRecord> reader = null;
      try {
        reader = new DataFileReader<GenericRecord>(file, datumReader);
        this.schema = reader.getSchema();
      } finally {
        Closeables.closeQuietly(reader);
      }
      return this;
    }

    /**
     * Configure the dataset's schema by using the schema from an existing Avro data
     * file. It is the caller's responsibility to close the {@link InputStream}.
     * A schema is required, and may be set using one of the <code>schema</code> or
     * <code>schemaFromAvroDataFile</code> methods.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schemaFromAvroDataFile(InputStream in) throws IOException {
      GenericDatumReader<GenericRecord> datumReader =
          new GenericDatumReader<GenericRecord>();
      DataFileStream<GenericRecord> stream = null;
      try {
        stream = new DataFileStream<GenericRecord>(in, datumReader);
        this.schema = stream.getSchema();
      } finally {
        Closeables.closeQuietly(stream);
      }
      return this;
    }

    /**
     * Configure the dataset's schema by using the schema from an existing Avro data
     * file.
     * A schema is required, and may be set using one of the <code>schema</code> or
     * <code>schemaFromAvroDataFile</code> methods.
     *
     * @return An instance of the builder for method chaining.
     */
    public Builder schemaFromAvroDataFile(URL url) throws IOException {
      InputStream in = null;
      try {
        in = url.openStream();
        return schemaFromAvroDataFile(in);
      } finally {
        Closeables.closeQuietly(in);
      }
    }

    /**
     * Configure the dataset's partitioning strategy. Optional.
     * 
     * @return An instance of the builder for method chaining.
{'fraction_non_alphanumeric': 0.09266442548056202, 'fraction_numerical': 0.0019825877079562106, 'mean_word_length': 2.8202173197234113, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7585753', 'n_tokens_mistral': 3198, 'n_tokens_neox': 3125, 'n_words': 1043}