<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.avro.file;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import org.apache.avro.util.Streams;

public class ZstandardCodec extends Codec {

  static class Option extends CodecFactory {
    private final int compressionLevel;
    private final boolean useChecksum;

    Option(int compressionLevel, boolean useChecksum) {
      this.compressionLevel = compressionLevel;
      this.useChecksum = useChecksum;
    }

    @Override
    protected Codec createInstance() {
      return new ZstandardCodec(compressionLevel, useChecksum);
    }
  }

  private final int compressionLevel;
  private final boolean useChecksum;
  private ByteArrayOutputStream outputBuffer;

  /**
   * Create a ZstandardCodec instance with the given compressionLevel and checksum
   * option
   **/
  public ZstandardCodec(int compressionLevel, boolean useChecksum) {
    this.compressionLevel = compressionLevel;
    this.useChecksum = useChecksum;
  }

  @Override
  public String getName() {
    return DataFileConstants.ZSTANDARD_CODEC;
  }

  @Override
  public ByteBuffer compress(ByteBuffer data) throws IOException {
    ByteArrayOutputStream baos = getOutputBuffer(data.remaining());
    try (OutputStream outputStream = ZstandardLoader.output(baos, compressionLevel, useChecksum)) {
      outputStream.write(data.array(), computeOffset(data), data.remaining());
    }
    return ByteBuffer.wrap(baos.toByteArray());
  }

  @Override
  public ByteBuffer decompress(ByteBuffer compressedData) throws IOException {
    ByteArrayOutputStream baos = getOutputBuffer(compressedData.remaining());
    InputStream bytesIn = new ByteArrayInputStream(compressedData.array(), computeOffset(compressedData),
        compressedData.remaining());
    try (InputStream ios = ZstandardLoader.input(bytesIn)) {
      Streams.copy(ios, baos);
    }
    return ByteBuffer.wrap(baos.toByteArray());
  }

  // get and initialize the output buffer for use.
  private ByteArrayOutputStream getOutputBuffer(int suggestedLength) {
    if (outputBuffer == null) {
      outputBuffer = new ByteArrayOutputStream(suggestedLength);
    }
    outputBuffer.reset();
    return outputBuffer;
  }

  @Override
  public int hashCode() {
    return getName().hashCode();
  }

  @Override
  public boolean equals(Object obj) {
    return (this == obj) || (obj != null && obj.getClass() == this.getClass());
  }

  @Override
  public String toString() {
    return getName() + "[" + compressionLevel + "]";
  }
}<commit_msg>[optimize] zstandard codec.<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.avro.file;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import org.apache.avro.util.ByteArrayBuilder;
import org.apache.avro.util.Streams;

public class ZstandardCodec extends Codec {

  static class Option extends CodecFactory {
    private final int compressionLevel;
    private final boolean useChecksum;

    Option(int compressionLevel, boolean useChecksum) {
      this.compressionLevel = compressionLevel;
      this.useChecksum = useChecksum;
    }

    @Override
    protected Codec createInstance() {
      return new ZstandardCodec(compressionLevel, useChecksum);
    }
  }

  private final int compressionLevel;
  private final boolean useChecksum;
  private ByteArrayBuilder outputBuffer;

  /**
   * Create a ZstandardCodec instance with the given compressionLevel and checksum
   * option
   **/
  public ZstandardCodec(int compressionLevel, boolean useChecksum) {
    this.compressionLevel = compressionLevel;
    this.useChecksum = useChecksum;
  }

  @Override
  public String getName() {
    return DataFileConstants.ZSTANDARD_CODEC;
  }

  @Override
  public ByteBuffer compress(ByteBuffer data) throws IOException {
    ByteArrayBuilder baos = getOutputBuffer(data.remaining());
    try (OutputStream outputStream = ZstandardLoader.output(baos, compressionLevel, useChecksum)) {
      outputStream.write(data.array(), computeOffset(data), data.remaining());
    }
    return ByteBuffer.wrap(baos.getBuffer(), 0, baos.size());
  }

  @Override
  public ByteBuffer decompress(ByteBuffer compressedData) throws IOException {
    ByteArrayBuilder baos = getOutputBuffer(compressedData.remaining());
    InputStream bytesIn = new ByteArrayInputStream(compressedData.array(), computeOffset(compressedData),
        compressedData.remaining());
    try (InputStream ios = ZstandardLoader.input(bytesIn)) {
      baos.readFrom(ios);
    }
    return ByteBuffer.wrap(baos.getBuffer(), 0, baos.size());
  }

  // get and initialize the output buffer for use.
  private ByteArrayBuilder getOutputBuffer(int suggestedLength) {
    if (outputBuffer == null) {
      outputBuffer = new ByteArrayBuilder(suggestedLength);
    }
    outputBuffer.reset();
    return outputBuffer;
  }

  @Override
  public int hashCode() {
    return getName().hashCode();
  }

  @Override
  public boolean equals(Object obj) {
    return (this == obj) || (obj != null && obj.getClass() == this.getClass());
  }

  @Override
  public String toString() {
    return getName() + "[" + compressionLevel + "]";
  }
}
{'fraction_non_alphanumeric': 0.08771174275050245, 'fraction_numerical': 0.001435544071202986, 'mean_word_length': 4.089116143170197, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4154854', 'n_tokens_mistral': 1869, 'n_tokens_neox': 1863, 'n_words': 589}