<commit_before>/**
 * Copyright 2010 <NAME>
 * 
 * 
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.hbasene.index;

import java.io.IOException;
import java.util.Map;
import java.util.NavigableMap;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTablePool;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Logger;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.util.Version;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;


/**
 * Abstract HBasene Test
 */
public class AbstractHBaseneTest {

  private static final Logger LOGGER = Logger
      .getLogger(AbstractHBaseneTest.class.getName());

  private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();

  protected Configuration conf;

  protected static final String TEST_INDEX = "idx-hbase-lucene";

  protected static final String PK_FIELD = "id";

  protected HTablePool  tablePool;
  
  protected static final int DEFAULT_POOL_SIZE = 20;
  
  /**
   * @throws java.lang.Exception
   */
  @BeforeClass
  public void setUpBeforeClass() throws Exception {
    TEST_UTIL.startMiniCluster(1);
    conf = TEST_UTIL.getConfiguration();
    HBaseIndexStore.dropLuceneIndexTable(TEST_INDEX, conf);
    HBaseIndexStore.createLuceneIndexTable(TEST_INDEX, conf, true);
    this.tablePool = new HTablePool(conf, DEFAULT_POOL_SIZE);
    HBaseIndexStore hbaseIndex = new HBaseIndexStore(this.tablePool, TEST_INDEX);

    HBaseIndexWriter writer = new HBaseIndexWriter(hbaseIndex, PK_FIELD);

    addDocument(writer, "FactTimes", "Messi plays for Barcelona");
    addDocument(writer, "UtopiaTimes", "Lionel M plays for Manchester United");
    addDocument(writer, "ThirdTimes", "Rooney plays for Manchester United");
    addDocument(
        writer,
        "FourthTimes",
        "Messi plays for argentina as well. He plays as a mid-fielder and plays really well.");

    Assert.assertTrue(new HBaseAdmin(conf).tableExists(TEST_INDEX));
  }

  /**
   * @throws java.lang.Exception
   */
  @AfterClass
  public void tearDownAfterClass() throws Exception {
    LOGGER.info("***   Shut down the HBase Cluster  ****");
    //this.tablePool.close();
    //TODO: HBASE-2435
    HBaseIndexStore.dropLuceneIndexTable(TEST_INDEX, conf);
    TEST_UTIL.shutdownMiniCluster();
  }

  protected void addDocument(final HBaseIndexWriter writer, final String id,
      final String content) throws CorruptIndexException, IOException {
    Document doc = new Document();
    doc.add(new Field("content", content, Field.Store.NO,
        Field.Index.ANALYZED_NO_NORMS));
    doc.add(new Field("id", id, Field.Store.YES, Field.Index.NO));
    writer.addDocument(doc, new StandardAnalyzer(Version.LUCENE_30));
  }

  protected void assertDocumentPresent(final String docId) throws IOException {
    Get get = new Get(Bytes.toBytes(docId));
    get.addFamily(HBaseneConstants.FAMILY_FIELDS);
    HTable table = new HTable(conf, TEST_INDEX);
    try {
      Result result = table.get(get);
      NavigableMap<byte[], byte[]> map = result
          .getFamilyMap(HBaseneConstants.FAMILY_FIELDS);
      Assert.assertTrue(map.size() > 0);
    } finally {
      table.close();
    }
  }

  protected void listAll(final byte[] family) throws IOException {<commit_msg>change in the underlying API - addDocument to createDocument<commit_after>/**
 * Copyright 2010 <NAME>
 * 
 * 
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.hbasene.index;

import java.io.IOException;
import java.util.Map;
import java.util.NavigableMap;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTablePool;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Logger;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.util.Version;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;

/**
 * Abstract HBasene Test
 */
public class AbstractHBaseneTest {

  private static final Logger LOGGER = Logger
      .getLogger(AbstractHBaseneTest.class.getName());

  private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();

  protected Configuration conf;

  protected static final String TEST_INDEX = "idx-hbase-lucene";

  protected static final String PK_FIELD = "id";

  protected HTablePool tablePool;

  protected static final int DEFAULT_POOL_SIZE = 20;

  /**
   * @throws java.lang.Exception
   */
  @BeforeClass
  public void setUpBeforeClass() throws Exception {
    TEST_UTIL.startMiniCluster(1);
    conf = TEST_UTIL.getConfiguration();
    HBaseIndexStore.dropLuceneIndexTable(TEST_INDEX, conf);
    HBaseIndexStore.createLuceneIndexTable(TEST_INDEX, conf, true);
    this.tablePool = new HTablePool(conf, DEFAULT_POOL_SIZE);
    HBaseIndexStore hbaseIndex = new HBaseIndexStore(this.tablePool, TEST_INDEX);

    HBaseIndexWriter writer = new HBaseIndexWriter(hbaseIndex, PK_FIELD);
    writer.addDocument(this.createDocument("FactTimes",
        "Messi plays for Barcelona"), new StandardAnalyzer(Version.LUCENE_30));
    writer.addDocument(this.createDocument( "UtopiaTimes",
        "Lionel M plays for Manchester United"), new StandardAnalyzer(
        Version.LUCENE_30));
    writer.addDocument(this.createDocument( "ThirdTimes",
        "Rooney plays for Manchester United"), new StandardAnalyzer(
        Version.LUCENE_30));
    writer
        .addDocument(
            this
                .createDocument(
                    "FourthTimes",
                    "Messi plays for argentina as well. He plays as a mid-fielder and plays really well."),
            new StandardAnalyzer(Version.LUCENE_30));

    Assert.assertTrue(new HBaseAdmin(conf).tableExists(TEST_INDEX));
  }

  /**
   * @throws java.lang.Exception
   */
  @AfterClass
  public void tearDownAfterClass() throws Exception {
    LOGGER.info("***   Shut down the HBase Cluster  ****");
    // this.tablePool.close();
    // TODO: HBASE-2435
    HBaseIndexStore.dropLuceneIndexTable(TEST_INDEX, conf);
    TEST_UTIL.shutdownMiniCluster();
  }

  protected Document createDocument(final String id, final String content)
      throws CorruptIndexException, IOException {
    Document doc = new Document();
    doc.add(new Field("content", content, Field.Store.NO,
        Field.Index.ANALYZED_NO_NORMS));
    doc.add(new Field("id", id, Field.Store.YES, Field.Index.NO));
    return doc;

  }

  protected void assertDocumentPresent(final String docId) throws IOException {
    Get get = new Get(Bytes.toBytes(docId));
    get.addFamily(HBaseneConstants.FAMILY_FIELDS);
    HTable table = new HTable(conf, TEST_INDEX);
    try {
      Result result = table.get(get);
      NavigableMap<byte[], byte[]> map = result
          .getFamilyMap(HBaseneConstants.FAMILY_FIELDS);
      Assert.assertTrue(map.size() > 0);
    } finally {
      table.close();
    }
  }

  protected void listAll(final byte[] family) throws IOException {
{'fraction_non_alphanumeric': 0.09772920203735144, 'fraction_numerical': 0.00466893039049236, 'mean_word_length': 4.048205677557579, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4552085', 'n_tokens_mistral': 2807, 'n_tokens_neox': 2716, 'n_words': 738}