<commit_before>/*
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hbase.io;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;

@Category(SmallTests.class)
public class TestHalfStoreFileReader {
  private static HBaseTestingUtility TEST_UTIL;

  @BeforeClass
  public static void setupBeforeClass() throws Exception {
    TEST_UTIL = new HBaseTestingUtility();
  }

  @AfterClass
  public static void tearDownAfterClass() throws Exception {
    TEST_UTIL.cleanupTestDir();
  }

  /**
   * Test the scanner and reseek of a half hfile scanner. The scanner API
   * demands that seekTo and reseekTo() only return < 0 if the key lies
   * before the start of the file (with no position on the scanner). Returning
   * 0 if perfect match (rare), and return > 1 if we got an imperfect match.
   *
   * The latter case being the most common, we should generally be returning 1,
   * and if we do, there may or may not be a 'next' in the scanner/file.
   *
   * A bug in the half file scanner was returning -1 at the end of the bottom
   * half, and that was causing the infrastructure above to go null causing NPEs
   * and other problems.  This test reproduces that failure, and also tests
   * both the bottom and top of the file while we are at it.
   *
   * @throws IOException
   */
  @Test
  public void testHalfScanAndReseek() throws IOException {
    String root_dir = TEST_UTIL.getDataTestDir("TestHalfStoreFile").toString();
    Path p = new Path(root_dir, "test");

    Configuration conf = TEST_UTIL.getConfiguration();
    FileSystem fs = FileSystem.get(conf);
    CacheConfig cacheConf = new CacheConfig(conf);

    HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
        .withPath(fs, p)
        .withBlockSize(1024)
        .withComparator(KeyValue.KEY_COMPARATOR)
        .create();

    // write some things.
    List<KeyValue> items = genSomeKeys();
    for (KeyValue kv : items) {
      w.append(kv);
    }
    w.close();

    HFile.Reader r = HFile.createReader(fs, p, cacheConf);
    r.loadFileInfo();
    byte [] midkey = r.midkey();
    KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
    midkey = midKV.getRow();

    //System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));

    Reference bottom = new Reference(midkey, Reference.Range.bottom);
    doTestOfScanAndReseek(p, fs, bottom, cacheConf);

    Reference top = new Reference(midkey, Reference.Range.top);
    doTestOfScanAndReseek(p, fs, top, cacheConf);

    r.close();
  }

  private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom,
      CacheConfig cacheConf)
      throws IOException {
    final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
        cacheConf, bottom, DataBlockEncoding.NONE);
    halfreader.loadFileInfo();
    final HFileScanner scanner = halfreader.getScanner(false, false);

    scanner.seekTo();
    KeyValue curr;
    do {
      curr = scanner.getKeyValue();
      KeyValue reseekKv =
          getLastOnCol(curr);
      int ret = scanner.reseekTo(reseekKv.getKey());
      assertTrue("reseek to returned: " + ret, ret > 0);
      //System.out.println(curr + ": " + ret);
    } while (scanner.next());

    int ret = scanner.reseekTo(getLastOnCol(curr).getKey());
    //System.out.println("Last reseek: " + ret);
    assertTrue( ret > 0 );

    halfreader.close(true);
  }


  // Tests the scanner on an HFile that is backed by HalfStoreFiles
  @Test
  public void testHalfScanner() throws IOException {
      String root_dir = TEST_UTIL.getDataTestDir("TestHalfStoreFileScanBefore").toString();
      Path p = new Path(root_dir, "test");
      Configuration conf = TEST_UTIL.getConfiguration();
      FileSystem fs = FileSystem.get(conf);<commit_msg>HBASE-7960 Addendum simplify test path as suggested by Stack<commit_after>/*
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hbase.io;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;

@Category(SmallTests.class)
public class TestHalfStoreFileReader {
  private static HBaseTestingUtility TEST_UTIL;

  @BeforeClass
  public static void setupBeforeClass() throws Exception {
    TEST_UTIL = new HBaseTestingUtility();
  }

  @AfterClass
  public static void tearDownAfterClass() throws Exception {
    TEST_UTIL.cleanupTestDir();
  }

  /**
   * Test the scanner and reseek of a half hfile scanner. The scanner API
   * demands that seekTo and reseekTo() only return < 0 if the key lies
   * before the start of the file (with no position on the scanner). Returning
   * 0 if perfect match (rare), and return > 1 if we got an imperfect match.
   *
   * The latter case being the most common, we should generally be returning 1,
   * and if we do, there may or may not be a 'next' in the scanner/file.
   *
   * A bug in the half file scanner was returning -1 at the end of the bottom
   * half, and that was causing the infrastructure above to go null causing NPEs
   * and other problems.  This test reproduces that failure, and also tests
   * both the bottom and top of the file while we are at it.
   *
   * @throws IOException
   */
  @Test
  public void testHalfScanAndReseek() throws IOException {
    String root_dir = TEST_UTIL.getDataTestDir().toString();
    Path p = new Path(root_dir, "test");

    Configuration conf = TEST_UTIL.getConfiguration();
    FileSystem fs = FileSystem.get(conf);
    CacheConfig cacheConf = new CacheConfig(conf);

    HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
        .withPath(fs, p)
        .withBlockSize(1024)
        .withComparator(KeyValue.KEY_COMPARATOR)
        .create();

    // write some things.
    List<KeyValue> items = genSomeKeys();
    for (KeyValue kv : items) {
      w.append(kv);
    }
    w.close();

    HFile.Reader r = HFile.createReader(fs, p, cacheConf);
    r.loadFileInfo();
    byte [] midkey = r.midkey();
    KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
    midkey = midKV.getRow();

    //System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));

    Reference bottom = new Reference(midkey, Reference.Range.bottom);
    doTestOfScanAndReseek(p, fs, bottom, cacheConf);

    Reference top = new Reference(midkey, Reference.Range.top);
    doTestOfScanAndReseek(p, fs, top, cacheConf);

    r.close();
  }

  private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom,
      CacheConfig cacheConf)
      throws IOException {
    final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
        cacheConf, bottom, DataBlockEncoding.NONE);
    halfreader.loadFileInfo();
    final HFileScanner scanner = halfreader.getScanner(false, false);

    scanner.seekTo();
    KeyValue curr;
    do {
      curr = scanner.getKeyValue();
      KeyValue reseekKv =
          getLastOnCol(curr);
      int ret = scanner.reseekTo(reseekKv.getKey());
      assertTrue("reseek to returned: " + ret, ret > 0);
      //System.out.println(curr + ": " + ret);
    } while (scanner.next());

    int ret = scanner.reseekTo(getLastOnCol(curr).getKey());
    //System.out.println("Last reseek: " + ret);
    assertTrue( ret > 0 );

    halfreader.close(true);
  }


  // Tests the scanner on an HFile that is backed by HalfStoreFiles
  @Test
  public void testHalfScanner() throws IOException {
      String root_dir = TEST_UTIL.getDataTestDir().toString();
      Path p = new Path(root_dir, "test");
      Configuration conf = TEST_UTIL.getConfiguration();
      FileSystem fs = FileSystem.get(conf);
{'fraction_non_alphanumeric': 0.09725542698670747, 'fraction_numerical': 0.0032514105383953332, 'mean_word_length': 3.6192579505300353, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5939741', 'n_tokens_mistral': 3116, 'n_tokens_neox': 3048, 'n_words': 985}