<commit_before>/**
 * Copyright 2013 Cloudera Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.cloudera.data.filesystem;

import com.cloudera.data.DatasetWriter;
import com.cloudera.data.DatasetWriterException;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.io.Closeables;
import java.io.Closeable;
import java.io.Flushable;
import java.io.IOException;
import org.apache.avro.Schema;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import parquet.avro.AvroParquetWriter;

class ParquetFileSystemDatasetWriter<E> implements DatasetWriter<E>, Flushable,
    Closeable {

  private static final Logger logger = LoggerFactory
    .getLogger(ParquetFileSystemDatasetWriter.class);

  private Path path;
  private Schema schema;
  private FileSystem fileSystem;

  private Path pathTmp;
  private AvroParquetWriter<E> avroParquetWriter;
  private ReaderWriterState state;

  public ParquetFileSystemDatasetWriter(FileSystem fileSystem, Path path,
      Schema schema) {
    this.fileSystem = fileSystem;
    this.path = path;
    this.pathTmp = new Path(path.getParent(), "." + path.getName() + ".tmp");
    this.schema = schema;
    this.state = ReaderWriterState.NEW;
  }

  @Override
  public void open() {
    Preconditions.checkState(state.equals(ReaderWriterState.NEW),
      "Unable to open a writer from state:%s", state);

    logger.debug(
      "Opening data file with pathTmp:{} (final path will be path:{})",
      pathTmp, path);

    try {
      avroParquetWriter = new AvroParquetWriter<E>(pathTmp.makeQualified(fileSystem),
          schema);
    } catch (IOException e) {
      throw new DatasetWriterException("Unable to create writer to path:" + pathTmp, e);
    }

    state = ReaderWriterState.OPEN;
  }

  @Override
  public void write(E entity) {
    Preconditions.checkState(state.equals(ReaderWriterState.OPEN),
      "Attempt to write to a writer in state:%s", state);

    try {
      avroParquetWriter.write(entity);
    } catch (IOException e) {
      throw new DatasetWriterException(
        "Unable to write entity:" + entity + " with writer:" + avroParquetWriter, e);
    }
  }
<commit_msg>CDK-31. Parquet format should use Snappy compression by default.<commit_after>/**
 * Copyright 2013 Cloudera Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.cloudera.data.filesystem;

import com.cloudera.data.DatasetWriter;
import com.cloudera.data.DatasetWriterException;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.io.Closeables;
import java.io.Closeable;
import java.io.Flushable;
import java.io.IOException;
import org.apache.avro.Schema;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.SnappyCodec;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import parquet.avro.AvroParquetWriter;
import parquet.hadoop.ParquetWriter;
import parquet.hadoop.metadata.CompressionCodecName;

class ParquetFileSystemDatasetWriter<E> implements DatasetWriter<E>, Flushable,
    Closeable {

  private static final Logger logger = LoggerFactory
    .getLogger(ParquetFileSystemDatasetWriter.class);

  private Path path;
  private Schema schema;
  private FileSystem fileSystem;
  private boolean enableCompression;

  private Path pathTmp;
  private AvroParquetWriter<E> avroParquetWriter;
  private ReaderWriterState state;

  public ParquetFileSystemDatasetWriter(FileSystem fileSystem, Path path,
      Schema schema) {
    this(fileSystem, path, schema, true);
  }

  public ParquetFileSystemDatasetWriter(FileSystem fileSystem, Path path,
      Schema schema, boolean enableCompression) {
    this.fileSystem = fileSystem;
    this.path = path;
    this.pathTmp = new Path(path.getParent(), "." + path.getName() + ".tmp");
    this.schema = schema;
    this.enableCompression = enableCompression;
    this.state = ReaderWriterState.NEW;
  }

  @Override
  public void open() {
    Preconditions.checkState(state.equals(ReaderWriterState.NEW),
      "Unable to open a writer from state:%s", state);

    logger.debug(
      "Opening data file with pathTmp:{} (final path will be path:{})",
      pathTmp, path);

    try {
      CompressionCodecName codecName = CompressionCodecName.UNCOMPRESSED;
      if (enableCompression) {
         if (SnappyCodec.isNativeCodeLoaded()) {
           codecName = CompressionCodecName.SNAPPY;
         } else {
           logger.warn("Compression enabled, but Snappy native code not loaded. " +
               "Parquet file will not be compressed.");
         }
      }
      avroParquetWriter = new AvroParquetWriter<E>(pathTmp.makeQualified(fileSystem),
          schema, codecName, ParquetWriter.DEFAULT_BLOCK_SIZE,
          ParquetWriter.DEFAULT_PAGE_SIZE);
    } catch (IOException e) {
      throw new DatasetWriterException("Unable to create writer to path:" + pathTmp, e);
    }

    state = ReaderWriterState.OPEN;
  }

  @Override
  public void write(E entity) {
    Preconditions.checkState(state.equals(ReaderWriterState.OPEN),
      "Attempt to write to a writer in state:%s", state);

    try {
      avroParquetWriter.write(entity);
    } catch (IOException e) {
      throw new DatasetWriterException(
        "Unable to write entity:" + entity + " with writer:" + avroParquetWriter, e);
    }
  }

{'fraction_non_alphanumeric': 0.09344211822660098, 'fraction_numerical': 0.0033866995073891628, 'mean_word_length': 3.8630239520958085, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5760551', 'n_tokens_mistral': 1904, 'n_tokens_neox': 1875, 'n_words': 543}