<commit_before>package com.continuuity.data.hbase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;

import java.io.File;
import java.io.IOException;
import java.util.Random;

/**
 * A base class that can be used to easily run a test within an embedded
 * HBase cluster (includes embedded ZooKeeper, HDFS, and HBase).
 *
 * To use, simply extend this class and create your tests like normal.  From
 * within your tests, you can access the underlying HBase cluster through
 * {@link #getConfiguration()}, {@link #getHBaseAdmin()}, and
 * {@link #getHTable(byte[])}.
 *
 * Alternatively, you can call the {@link #startHBase()} and {@link #stopHBase()}
 * methods directly from within your own BeforeClass/AfterClass methods.
 *
 * Note:  This test is somewhat heavy-weight and takes 10-20 seconds to startup.
 */
public abstract class HBaseTestBase {

  protected static Configuration conf;

  protected static MiniZooKeeperCluster zkCluster;

  protected static MiniDFSCluster dfsCluster;

  protected static MiniHBaseCluster hbaseCluster;

  // Accessors for test implementations

  public static Configuration getConfiguration() {
    return conf;
  }

  public static HBaseAdmin getHBaseAdmin() throws IOException {
    return new HBaseAdmin(conf);
  }

  public static HTable getHTable(byte [] tableName) throws IOException {
    return new HTable(conf, tableName);
  }

  // Temporary directories

  private static final Random r = new Random();

  public static File getRandomTempDir() {
    File file = new File(System.getProperty("java.io.tmpdir"),
        Integer.toString(Math.abs(r.nextInt())));
    if (!file.mkdir()) {
      throw new RuntimeException("Unable to create temp directory");
    }
    return file;
  }

  // Test startup / teardown

  @BeforeClass
  public static void startHBase() throws Exception {
    conf = new Configuration();
    // Set any necessary configurations (disable UIs to prevent port conflicts)
    conf.setInt("hbase.regionserver.info.port", -1);
    conf.setInt("hbase.master.info.port", -1);

    // Start ZooKeeper

    zkCluster = new MiniZooKeeperCluster(conf);
    System.err.println("Starting ZK in 1 sec...");
    Thread.sleep(1000);
    int zkPort = zkCluster.startup(getRandomTempDir(), 1);
    System.err.println("\n\nStarted ZK on port " + zkPort + "\n\n\n");

    // Add ZK info to conf
    conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(zkPort));

    // Start DFS

    File dfsPath = getRandomTempDir();
    System.setProperty("test.build.data", dfsPath.toString());
    System.setProperty("test.cache.data", dfsPath.toString());
    System.err.println("Instantiating dfs cluster in 1 sec...");
    Thread.sleep(1000);
    dfsCluster = new MiniDFSCluster.Builder(conf)
        .nameNodePort(0)
        .numDataNodes(1)
        .format(true)
        .manageDataDfsDirs(true)
        .manageNameDfsDirs(true)
        .build();
    System.err.println("Waiting for dfs to start...");
    dfsCluster.waitClusterUp();
    System.err.println("DFS started...");
    Thread.sleep(1000);

    // Add HDFS info to conf
    conf.set("fs.defaultFS", dfsCluster.getFileSystem().getUri().toString());

    // Start HBase
    createHBaseRootDir(conf);
    conf.setInt("hbase.master.wait.on.regionservers.mintostart", 1);
    conf.setInt("hbase.master.wait.on.regionservers.maxtostart", 1);
    Configuration c = new Configuration(conf);
    System.err.println("Instantiating HBase cluster in 1 sec...");
    Thread.sleep(1000);
    hbaseCluster = new MiniHBaseCluster(c, 1, 1);
    System.err.println("Just waiting around for a bit now");
    Thread.sleep(1000);
  }

  @AfterClass
  public static void stopHBase() throws Exception {<commit_msg>Increase zookeeper timeouts for data-fabric tests<commit_after>package com.continuuity.data.hbase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;

import java.io.File;
import java.io.IOException;
import java.util.Random;

/**
 * A base class that can be used to easily run a test within an embedded
 * HBase cluster (includes embedded ZooKeeper, HDFS, and HBase).
 *
 * To use, simply extend this class and create your tests like normal.  From
 * within your tests, you can access the underlying HBase cluster through
 * {@link #getConfiguration()}, {@link #getHBaseAdmin()}, and
 * {@link #getHTable(byte[])}.
 *
 * Alternatively, you can call the {@link #startHBase()} and {@link #stopHBase()}
 * methods directly from within your own BeforeClass/AfterClass methods.
 *
 * Note:  This test is somewhat heavy-weight and takes 10-20 seconds to startup.
 */
public abstract class HBaseTestBase {

  protected static Configuration conf;

  protected static MiniZooKeeperCluster zkCluster;

  protected static MiniDFSCluster dfsCluster;

  protected static MiniHBaseCluster hbaseCluster;

  // Accessors for test implementations

  public static Configuration getConfiguration() {
    return conf;
  }

  public static HBaseAdmin getHBaseAdmin() throws IOException {
    return new HBaseAdmin(conf);
  }

  public static HTable getHTable(byte [] tableName) throws IOException {
    return new HTable(conf, tableName);
  }

  // Temporary directories

  private static final Random r = new Random();

  public static File getRandomTempDir() {
    File file = new File(System.getProperty("java.io.tmpdir"),
        Integer.toString(Math.abs(r.nextInt())));
    if (!file.mkdir()) {
      throw new RuntimeException("Unable to create temp directory");
    }
    return file;
  }

  // Test startup / teardown

  @BeforeClass
  public static void startHBase() throws Exception {
    conf = new Configuration();
    // Set any necessary configurations (disable UIs to prevent port conflicts)
    conf.setInt("hbase.regionserver.info.port", -1);
    conf.setInt("hbase.master.info.port", -1);

    // Start ZooKeeper

    zkCluster = new MiniZooKeeperCluster(conf);
    System.err.println("Starting ZK in 1 sec...");
    Thread.sleep(1000);
    int zkPort = zkCluster.startup(getRandomTempDir(), 1);
    System.err.println("\n\nStarted ZK on port " + zkPort + "\n\n\n");

    // Add ZK info to conf
    conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(zkPort));

    // Start DFS

    File dfsPath = getRandomTempDir();
    System.setProperty("test.build.data", dfsPath.toString());
    System.setProperty("test.cache.data", dfsPath.toString());
    System.err.println("Instantiating dfs cluster in 1 sec...");
    Thread.sleep(1000);
    dfsCluster = new MiniDFSCluster.Builder(conf)
        .nameNodePort(0)
        .numDataNodes(1)
        .format(true)
        .manageDataDfsDirs(true)
        .manageNameDfsDirs(true)
        .build();
    System.err.println("Waiting for dfs to start...");
    dfsCluster.waitClusterUp();
    System.err.println("DFS started...");
    Thread.sleep(1000);

    // Add HDFS info to conf
    conf.set("fs.defaultFS", dfsCluster.getFileSystem().getUri().toString());

    // Start HBase
    createHBaseRootDir(conf);
    conf.setInt("hbase.master.wait.on.regionservers.mintostart", 1);
    conf.setInt("hbase.master.wait.on.regionservers.maxtostart", 1);
    conf.setInt("zookeeper.session.timeout", 300000); // increasing session timeout for unit tests
    Configuration c = new Configuration(conf);
    System.err.println("Instantiating HBase cluster in 1 sec...");
    Thread.sleep(1000);
    hbaseCluster = new MiniHBaseCluster(c, 1, 1);
    System.err.println("Just waiting around for a bit now");
    Thread.sleep(1000);
  }

  @AfterClass
  public static void stopHBase() throws Exception {
{'fraction_non_alphanumeric': 0.11804981702278361, 'fraction_numerical': 0.009207885727777122, 'mean_word_length': 3.8, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3400320', 'n_tokens_mistral': 2666, 'n_tokens_neox': 2542, 'n_words': 671}