<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.io.hfile;

import java.io.DataOutputStream;
import java.io.IOException;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
import org.apache.hadoop.hbase.security.EncryptionUtil;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.WritableUtils;

/**
 * {@link HFile} writer for version 3.
 */
@InterfaceAudience.Private
public class HFileWriterV3 extends HFileWriterV2 {

  private static final Log LOG = LogFactory.getLog(HFileWriterV3.class);

  private int maxTagsLength = 0;

  static class WriterFactoryV3 extends HFile.WriterFactory {
    WriterFactoryV3(Configuration conf, CacheConfig cacheConf) {
      super(conf, cacheConf);
    }

    @Override
    public Writer createWriter(FileSystem fs, Path path, FSDataOutputStream ostream,
        final KVComparator comparator, HFileContext fileContext)
        throws IOException {
      return new HFileWriterV3(conf, cacheConf, fs, path, ostream, comparator, fileContext);
    }
  }

  /** Constructor that takes a path, creates and closes the output stream. */
  public HFileWriterV3(Configuration conf, CacheConfig cacheConf, FileSystem fs, Path path,
      FSDataOutputStream ostream, final KVComparator comparator,
      final HFileContext fileContext) throws IOException {
    super(conf, cacheConf, fs, path, ostream, comparator, fileContext);
    if (LOG.isTraceEnabled()) {
      LOG.trace("Writer" + (path != null ? " for " + path : "") +
        " initialized with cacheConf: " + cacheConf +
        " comparator: " + comparator.getClass().getSimpleName() +
        " fileContext: " + fileContext);
    }
  }

  /**
   * Add key/value to file. Keys must be added in an order that agrees with the
   * Comparator passed on construction.
   * 
   * @param kv
   *          KeyValue to add. Cannot be empty nor null.
   * @throws IOException
   */
  @Override
  public void append(final KeyValue kv) throws IOException {
    // Currently get the complete arrays
    append(kv.getMvccVersion(), kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(),
        kv.getValueArray(), kv.getValueOffset(), kv.getValueLength(), kv.getTagsArray(),
        kv.getTagsOffset(), kv.getTagsLength());
    this.maxMemstoreTS = Math.max(this.maxMemstoreTS, kv.getMvccVersion());
  }
  
  /**
   * Add key/value to file. Keys must be added in an order that agrees with the
   * Comparator passed on construction.
   * @param key
   *          Key to add. Cannot be empty nor null.
   * @param value
   *          Value to add. Cannot be empty nor null.
   * @throws IOException
   */
  @Override
  public void append(final byte[] key, final byte[] value) throws IOException {
    append(key, value, HConstants.EMPTY_BYTE_ARRAY);
  }

  /**
   * Add key/value to file. Keys must be added in an order that agrees with the
   * Comparator passed on construction.
   * @param key
   *          Key to add. Cannot be empty nor null.
   * @param value
   *          Value to add. Cannot be empty nor null.
   * @param tag
   *          Tag t add. Cannot be empty or null.
   * @throws IOException
   */
  @Override
  public void append(final byte[] key, final byte[] value, byte[] tag) throws IOException {
    append(0, key, 0, key.length, value, 0, value.length, tag, 0, tag.length);
  }

  /**
   * Add key/value to file. Keys must be added in an order that agrees with the
   * Comparator passed on construction.
   * @param key
   * @param koffset
   * @param klength
   * @param value
   * @param voffset
   * @param vlength
   * @param tag
   * @param tagsOffset
   * @param tagLength
   * @throws IOException
   */
  private void append(final long memstoreTS, final byte[] key, final int koffset,
      final int klength, final byte[] value, final int voffset, final int vlength,
      final byte[] tag, final int tagsOffset, final int tagsLength) throws IOException {
    boolean dupKey = checkKey(key, koffset, klength);
    checkValue(value, voffset, vlength);
    if (!dupKey) {
      checkBlockBoundary();
    }

    if (!fsBlockWriter.isWriting())
      newBlock();

    // Write length of key and value and then actual key and value bytes.
    // Additionally, we may also write down the memstoreTS.
    {
      DataOutputStream out = fsBlockWriter.getUserDataStream();
      out.writeInt(klength);
      totalKeyLength += klength;
      out.writeInt(vlength);
      totalValueLength += vlength;
      out.write(key, koffset, klength);
      out.write(value, voffset, vlength);
      // Write the additional tag into the stream
      if (hFileContext.isIncludesTags()) {
        out.writeShort((short) tagsLength);
        if (tagsLength > 0) {
          out.write(tag, tagsOffset, tagsLength);
          if (tagsLength > maxTagsLength) {
            maxTagsLength = tagsLength;
          }
        }
      }
      if (this.hFileContext.isIncludesMvcc()) {
        WritableUtils.writeVLong(out, memstoreTS);
      }
    }

    // Are we the first key in this block?
    if (firstKeyInBlock == null) {
      // Copy the key.
      firstKeyInBlock = new byte[klength];
      System.arraycopy(key, koffset, firstKeyInBlock, 0, klength);
    }

    lastKeyBuffer = key;
    lastKeyOffset = koffset;
    lastKeyLength = klength;
    entryCount++;
  }
  
  protected void finishFileInfo() throws IOException {
    super.finishFileInfo();
    if (hFileContext.isIncludesTags()) {
      // When tags are not being written in this file, MAX_TAGS_LEN is excluded
      // from the FileInfo
      fileInfo.append(FileInfo.MAX_TAGS_LEN, Bytes.toBytes(this.maxTagsLength), false);
      boolean tagsCompressed = (hFileContext.getDataBlockEncoding() != DataBlockEncoding.NONE)
        && hFileContext.isCompressTags();
      fileInfo.append(FileInfo.TAGS_COMPRESSED, Bytes.toBytes(tagsCompressed), false);
    }
  }

  @Override<commit_msg>HBASE-10454 Tags presence file info can be wrong in HFiles when PrefixTree encoding is used.<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.io.hfile;

import java.io.DataOutputStream;
import java.io.IOException;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
import org.apache.hadoop.hbase.security.EncryptionUtil;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.WritableUtils;

/**
 * {@link HFile} writer for version 3.
 */
@InterfaceAudience.Private
public class HFileWriterV3 extends HFileWriterV2 {

  private static final Log LOG = LogFactory.getLog(HFileWriterV3.class);

  private int maxTagsLength = 0;

  static class WriterFactoryV3 extends HFile.WriterFactory {
    WriterFactoryV3(Configuration conf, CacheConfig cacheConf) {
      super(conf, cacheConf);
    }

    @Override
    public Writer createWriter(FileSystem fs, Path path, FSDataOutputStream ostream,
        final KVComparator comparator, HFileContext fileContext)
        throws IOException {
      return new HFileWriterV3(conf, cacheConf, fs, path, ostream, comparator, fileContext);
    }
  }

  /** Constructor that takes a path, creates and closes the output stream. */
  public HFileWriterV3(Configuration conf, CacheConfig cacheConf, FileSystem fs, Path path,
      FSDataOutputStream ostream, final KVComparator comparator,
      final HFileContext fileContext) throws IOException {
    super(conf, cacheConf, fs, path, ostream, comparator, fileContext);
    if (LOG.isTraceEnabled()) {
      LOG.trace("Writer" + (path != null ? " for " + path : "") +
        " initialized with cacheConf: " + cacheConf +
        " comparator: " + comparator.getClass().getSimpleName() +
        " fileContext: " + fileContext);
    }
  }

  /**
   * Add key/value to file. Keys must be added in an order that agrees with the
   * Comparator passed on construction.
   * 
   * @param kv
   *          KeyValue to add. Cannot be empty nor null.
   * @throws IOException
   */
  @Override
  public void append(final KeyValue kv) throws IOException {
    // Currently get the complete arrays
    append(kv.getMvccVersion(), kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(),
        kv.getValueArray(), kv.getValueOffset(), kv.getValueLength(), kv.getTagsArray(),
        kv.getTagsOffset(), kv.getTagsLength());
    this.maxMemstoreTS = Math.max(this.maxMemstoreTS, kv.getMvccVersion());
  }
  
  /**
   * Add key/value to file. Keys must be added in an order that agrees with the
   * Comparator passed on construction.
   * @param key
   *          Key to add. Cannot be empty nor null.
   * @param value
   *          Value to add. Cannot be empty nor null.
   * @throws IOException
   */
  @Override
  public void append(final byte[] key, final byte[] value) throws IOException {
    append(key, value, HConstants.EMPTY_BYTE_ARRAY);
  }

  /**
   * Add key/value to file. Keys must be added in an order that agrees with the
   * Comparator passed on construction.
   * @param key
   *          Key to add. Cannot be empty nor null.
   * @param value
   *          Value to add. Cannot be empty nor null.
   * @param tag
   *          Tag t add. Cannot be empty or null.
   * @throws IOException
   */
  @Override
  public void append(final byte[] key, final byte[] value, byte[] tag) throws IOException {
    append(0, key, 0, key.length, value, 0, value.length, tag, 0, tag.length);
  }

  /**
   * Add key/value to file. Keys must be added in an order that agrees with the
   * Comparator passed on construction.
   * @param key
   * @param koffset
   * @param klength
   * @param value
   * @param voffset
   * @param vlength
   * @param tag
   * @param tagsOffset
   * @param tagLength
   * @throws IOException
   */
  private void append(final long memstoreTS, final byte[] key, final int koffset,
      final int klength, final byte[] value, final int voffset, final int vlength,
      final byte[] tag, final int tagsOffset, final int tagsLength) throws IOException {
    boolean dupKey = checkKey(key, koffset, klength);
    checkValue(value, voffset, vlength);
    if (!dupKey) {
      checkBlockBoundary();
    }

    if (!fsBlockWriter.isWriting())
      newBlock();

    // Write length of key and value and then actual key and value bytes.
    // Additionally, we may also write down the memstoreTS.
    {
      DataOutputStream out = fsBlockWriter.getUserDataStream();
      out.writeInt(klength);
      totalKeyLength += klength;
      out.writeInt(vlength);
      totalValueLength += vlength;
      out.write(key, koffset, klength);
      out.write(value, voffset, vlength);
      // Write the additional tag into the stream
      if (hFileContext.isIncludesTags()) {
        out.writeShort((short) tagsLength);
        if (tagsLength > 0) {
          out.write(tag, tagsOffset, tagsLength);
          if (tagsLength > maxTagsLength) {
            maxTagsLength = tagsLength;
          }
        }
      }
      if (this.hFileContext.isIncludesMvcc()) {
        WritableUtils.writeVLong(out, memstoreTS);
      }
    }

    // Are we the first key in this block?
    if (firstKeyInBlock == null) {
      // Copy the key.
      firstKeyInBlock = new byte[klength];
      System.arraycopy(key, koffset, firstKeyInBlock, 0, klength);
    }

    lastKeyBuffer = key;
    lastKeyOffset = koffset;
    lastKeyLength = klength;
    entryCount++;
  }
  
  protected void finishFileInfo() throws IOException {
    super.finishFileInfo();
    if (hFileContext.getDataBlockEncoding() == DataBlockEncoding.PREFIX_TREE) {
      // In case of Prefix Tree encoding, we always write tags information into HFiles even if all
      // KVs are having no tags.
      fileInfo.append(FileInfo.MAX_TAGS_LEN, Bytes.toBytes(this.maxTagsLength), false);
    } else if (hFileContext.isIncludesTags()) {
      // When tags are not being written in this file, MAX_TAGS_LEN is excluded
      // from the FileInfo
      fileInfo.append(FileInfo.MAX_TAGS_LEN, Bytes.toBytes(this.maxTagsLength), false);
      boolean tagsCompressed = (hFileContext.getDataBlockEncoding() != DataBlockEncoding.NONE)
        && hFileContext.isCompressTags();
      fileInfo.append(FileInfo.TAGS_COMPRESSED, Bytes.toBytes(tagsCompressed), false);
    }
  }

  @Override
{'fraction_non_alphanumeric': 0.0967935071846727, 'fraction_numerical': 0.0028605641298563066, 'mean_word_length': 3.233455364686004, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '682230', 'n_tokens_mistral': 4374, 'n_tokens_neox': 4295, 'n_words': 1381}