<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hive.druid;

import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.service.AbstractService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

/**
 * This class has the hooks to start and stop the external Druid Nodes
 */
public class MiniDruidCluster extends AbstractService {
  private static final Logger log = LoggerFactory.getLogger(MiniDruidCluster.class);

  private static final String COMMON_DRUID_JVM_PROPPERTIES = "-Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Ddruid.emitter=logging -Ddruid.emitter.logging.logLevel=info";

  private static final List<String> HISTORICAL_JVM_CONF = Arrays
          .asList("-server", "-XX:MaxDirectMemorySize=10g", "-Xmx512m", "-Xmx512m",
                  COMMON_DRUID_JVM_PROPPERTIES
          );

  private static final List<String> COORDINATOR_JVM_CONF = Arrays
          .asList("-server", "-XX:MaxDirectMemorySize=2g", "-Xmx512m", "-Xms512m",
                  COMMON_DRUID_JVM_PROPPERTIES
          );

  private static final Map<String, String> COMMON_DRUID_CONF = ImmutableMap.of(
          "druid.metadata.storage.type", "derby",
          "druid.storage.type", "hdfs",
          "druid.processing.buffer.sizeBytes", "213870912",
          "druid.processing.numThreads", "2"
  );

  private static final Map<String, String> COMMON_DRUID_HISTORICAL = ImmutableMap.of(
          "druid.server.maxSize", "130000000000"
  );

  private static final Map<String, String> COMMON_COORDINATOR_INDEXER = ImmutableMap
          .of(
                  "druid.indexer.logs.type", "file",
                  "druid.coordinator.asOverlord.enabled", "true",
                  "druid.coordinator.asOverlord.overlordService", "druid/overlord",
                  "druid.coordinator.period", "PT10S",
                  "druid.manager.segments.pollDuration", "PT10S"
          );

  private final DruidNode historical;

  private final DruidNode broker;

  // Coordinator is running as Overlord as well.
  private final DruidNode coordinator;

  private final List<DruidNode> druidNodes;

  private final File dataDirectory;

  private final File logDirectory;

  public MiniDruidCluster(String name) {
    this(name, "/tmp/miniDruid/log", "/tmp/miniDruid/data", 2181, null);
  }


  public MiniDruidCluster(String name, String logDir, String tmpDir, Integer zookeeperPort, String classpath) {
    super(name);
    this.dataDirectory = new File(tmpDir, "druid-data");
    this.logDirectory = new File(logDir);

    ensureCleanDirectory(dataDirectory);

    String derbyURI = String
            .format("jdbc:derby://localhost:1527/%s/druid_derby/metadata.db;create=true",
                    dataDirectory.getAbsolutePath()
            );
    String segmentsCache = String
            .format("[{\"path\":\"%s/druid/segment-cache\",\"maxSize\":130000000000}]",
                    dataDirectory.getAbsolutePath()
            );
    String indexingLogDir = new File(logDirectory, "indexer-log").getAbsolutePath();

    ImmutableMap.Builder<String, String> coordinatorMapBuilder = new ImmutableMap.Builder();
    ImmutableMap.Builder<String, String> historicalMapBuilder = new ImmutableMap.Builder();

    Map<String, String> coordinatorProperties = coordinatorMapBuilder.putAll(COMMON_DRUID_CONF)
            .putAll(COMMON_COORDINATOR_INDEXER)
            .put("druid.metadata.storage.connector.connectURI", derbyURI)
            .put("druid.indexer.logs.directory", indexingLogDir)
            .put("druid.zk.service.host", "localhost:" + zookeeperPort)
            .put("druid.coordinator.startDelay", "PT1S")
            .put("druid.indexer.runner", "local")
            .put("druid.storage.storageDirectory", getDeepStorageDir())
            .build();
    Map<String, String> historicalProperties = historicalMapBuilder.putAll(COMMON_DRUID_CONF)
            .putAll(COMMON_DRUID_HISTORICAL)
            .put("druid.zk.service.host", "localhost:" + zookeeperPort)
            .put("druid.segmentCache.locations", segmentsCache)
            .put("druid.storage.storageDirectory", getDeepStorageDir())
            .build();
    coordinator = new ForkingDruidNode("coordinator", classpath, coordinatorProperties,
            COORDINATOR_JVM_CONF,
            logDirectory, null
    );
    historical = new ForkingDruidNode("historical", classpath, historicalProperties, HISTORICAL_JVM_CONF,
            logDirectory, null
    );
    broker = new ForkingDruidNode("broker", classpath, historicalProperties, HISTORICAL_JVM_CONF,
            logDirectory, null
    );
    druidNodes = Arrays.asList(coordinator, historical, broker);

  }

  private static void ensureCleanDirectory(File dir){
    try {
      if (dir.exists()) {
        // need to clean data directory to ensure that there is no interference from old runs
        // Cleaning is happening here to allow debugging in case of tests fail
        // we don;t have to clean logs since it is an append mode
        log.info("Cleaning the druid directory [{}]", dir.getAbsolutePath());
        FileUtils.deleteDirectory(dir);
      } else {
        log.info("Creating the druid directory [{}]", dir.getAbsolutePath());
        dir.mkdirs();
      }
    } catch (IOException e) {
      log.error("Failed to clean druid directory");
      Throwables.propagate(e);
    }
  }

  @Override
  protected void serviceStart() throws Exception {
    druidNodes.stream().forEach(node -> {
      try {
        node.start();
      } catch (IOException e) {
        log.error("Failed to start node " + node.getNodeType()
                + " Consequently will destroy the cluster");
        druidNodes.stream().filter(node1 -> node1.isAlive()).forEach(nodeToStop -> {
          try {
            log.info("Stopping Node " + nodeToStop.getNodeType());
            nodeToStop.close();
          } catch (IOException e1) {
            log.error("Error while stopping " + nodeToStop.getNodeType(), e1);
          }
        });
        Throwables.propagate(e);
      }
    });
  }

  @Override
  protected void serviceStop() throws Exception {
    druidNodes.stream().forEach(node -> {
      try {
        node.close();
      } catch (IOException e) {
        // nothing that we can really do about it
        log.error(String.format("Failed to stop druid node [%s]", node.getNodeType()), e);
      }
    });
  }


  public String getMetadataURI() {
    return String.format("jdbc:derby://localhost:1527/%s/druid_derby/metadata.db",
            dataDirectory.getAbsolutePath()
    );
  }

  public String getDeepStorageDir() {
    return dataDirectory.getAbsolutePath() + File.separator + "deep-storage";
  }

  public String getCoordinatorURI(){
    return "localhost:8081";
  }

  public String getOverlordURI(){
    // Overlord and coordinator both run in same JVM.
    return getCoordinatorURI();
  }
}<commit_msg>HIVE-20425 : Use a custom range of port for embedded Derby used by Druid. (Slim Bouguerra via <NAME>)<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hive.druid;

import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.service.AbstractService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.net.DatagramSocket;
import java.net.ServerSocket;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

/**
 * This class has the hooks to start and stop the external Druid Nodes
 */
public class MiniDruidCluster extends AbstractService {
  private static final Logger log = LoggerFactory.getLogger(MiniDruidCluster.class);

  private static final String COMMON_DRUID_JVM_PROPPERTIES = "-Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Ddruid.emitter=logging -Ddruid.emitter.logging.logLevel=info";

  private static final List<String> HISTORICAL_JVM_CONF = Arrays
          .asList("-server", "-XX:MaxDirectMemorySize=10g", "-Xmx512m", "-Xmx512m",
                  COMMON_DRUID_JVM_PROPPERTIES
          );

  private static final List<String> COORDINATOR_JVM_CONF = Arrays
          .asList("-server", "-XX:MaxDirectMemorySize=2g", "-Xmx512m", "-Xms512m",
                  COMMON_DRUID_JVM_PROPPERTIES
          );

  private static final Map<String, String> COMMON_DRUID_CONF = ImmutableMap.of(
          "druid.metadata.storage.type", "derby",
          "druid.storage.type", "hdfs",
          "druid.processing.buffer.sizeBytes", "213870912",
          "druid.processing.numThreads", "2"
  );

  private static final Map<String, String> COMMON_DRUID_HISTORICAL = ImmutableMap.of(
          "druid.server.maxSize", "130000000000"
  );

  private static final Map<String, String> COMMON_COORDINATOR_INDEXER = ImmutableMap
          .of(
                  "druid.indexer.logs.type", "file",
                  "druid.coordinator.asOverlord.enabled", "true",
                  "druid.coordinator.asOverlord.overlordService", "druid/overlord",
                  "druid.coordinator.period", "PT10S",
                  "druid.manager.segments.pollDuration", "PT10S"
          );
  private static final int MIN_PORT_NUMBER = 60000;
  private static final int MAX_PORT_NUMBER = 65535;

  private final DruidNode historical;

  private final DruidNode broker;

  // Coordinator is running as Overlord as well.
  private final DruidNode coordinator;

  private final List<DruidNode> druidNodes;

  private final File dataDirectory;

  private final File logDirectory;
  private final String derbyURI;

  public MiniDruidCluster(String name) {
    this(name, "/tmp/miniDruid/log", "/tmp/miniDruid/data", 2181, null);
  }


  public MiniDruidCluster(String name, String logDir, String tmpDir, Integer zookeeperPort, String classpath) {
    super(name);
    this.dataDirectory = new File(tmpDir, "druid-data");
    this.logDirectory = new File(logDir);
    int derbyPort = findPort(MIN_PORT_NUMBER, MAX_PORT_NUMBER);


    ensureCleanDirectory(dataDirectory);

    derbyURI = String
            .format("jdbc:derby://localhost:%s/%s/druid_derby/metadata.db;create=true",
                    derbyPort,
                    dataDirectory.getAbsolutePath()
            );
    String segmentsCache = String
            .format("[{\"path\":\"%s/druid/segment-cache\",\"maxSize\":130000000000}]",
                    dataDirectory.getAbsolutePath()
            );
    String indexingLogDir = new File(logDirectory, "indexer-log").getAbsolutePath();

    ImmutableMap.Builder<String, String> coordinatorMapBuilder = new ImmutableMap.Builder();
    ImmutableMap.Builder<String, String> historicalMapBuilder = new ImmutableMap.Builder();

    Map<String, String> coordinatorProperties = coordinatorMapBuilder.putAll(COMMON_DRUID_CONF)
            .putAll(COMMON_COORDINATOR_INDEXER)
            .put("druid.metadata.storage.connector.connectURI", derbyURI)
            .put("druid.metadata.storage.connector.port", String.valueOf(derbyPort))
            .put("druid.indexer.logs.directory", indexingLogDir)
            .put("druid.zk.service.host", "localhost:" + zookeeperPort)
            .put("druid.coordinator.startDelay", "PT1S")
            .put("druid.indexer.runner", "local")
            .put("druid.storage.storageDirectory", getDeepStorageDir())
            .build();
    Map<String, String> historicalProperties = historicalMapBuilder.putAll(COMMON_DRUID_CONF)
            .putAll(COMMON_DRUID_HISTORICAL)
            .put("druid.zk.service.host", "localhost:" + zookeeperPort)
            .put("druid.segmentCache.locations", segmentsCache)
            .put("druid.storage.storageDirectory", getDeepStorageDir())
            .build();
    coordinator = new ForkingDruidNode("coordinator", classpath, coordinatorProperties,
            COORDINATOR_JVM_CONF,
            logDirectory, null
    );
    historical = new ForkingDruidNode("historical", classpath, historicalProperties, HISTORICAL_JVM_CONF,
            logDirectory, null
    );
    broker = new ForkingDruidNode("broker", classpath, historicalProperties, HISTORICAL_JVM_CONF,
            logDirectory, null
    );
    druidNodes = Arrays.asList(coordinator, historical, broker);

  }

  private int findPort(int start, int end) {
    int port = start;
    while (!available(port)) {
      port++;
      if (port == end) {
        throw  new RuntimeException("can not find free port for range " + start + ":" + end);
      }
    }
    return port;
  }

  /**
   * Checks to see if a specific port is available.
   *
   * @param port the port to check for availability
   */
  public static boolean available(int port) {
    if (port < MIN_PORT_NUMBER || port > MAX_PORT_NUMBER) {
      throw new IllegalArgumentException("Invalid start port: " + port);
    }

    ServerSocket ss = null;
    DatagramSocket ds = null;
    try {
      ss = new ServerSocket(port);
      ss.setReuseAddress(true);
      ds = new DatagramSocket(port);
      ds.setReuseAddress(true);
      return true;
    } catch (IOException e) {
    } finally {
      if (ds != null) {
        ds.close();
      }

      if (ss != null) {
        try {
          ss.close();
        } catch (IOException e) {
          /* should not be thrown */
        }
      }
    }

    return false;
  }

  private static void ensureCleanDirectory(File dir){
    try {
      if (dir.exists()) {
        // need to clean data directory to ensure that there is no interference from old runs
        // Cleaning is happening here to allow debugging in case of tests fail
        // we don;t have to clean logs since it is an append mode
        log.info("Cleaning the druid directory [{}]", dir.getAbsolutePath());
        FileUtils.deleteDirectory(dir);
      } else {
        log.info("Creating the druid directory [{}]", dir.getAbsolutePath());
        dir.mkdirs();
      }
    } catch (IOException e) {
      log.error("Failed to clean druid directory");
      Throwables.propagate(e);
    }
  }

  @Override
  protected void serviceStart() throws Exception {
    druidNodes.stream().forEach(node -> {
      try {
        node.start();
      } catch (IOException e) {
        log.error("Failed to start node " + node.getNodeType()
                + " Consequently will destroy the cluster");
        druidNodes.stream().filter(node1 -> node1.isAlive()).forEach(nodeToStop -> {
          try {
            log.info("Stopping Node " + nodeToStop.getNodeType());
            nodeToStop.close();
          } catch (IOException e1) {
            log.error("Error while stopping " + nodeToStop.getNodeType(), e1);
          }
        });
        Throwables.propagate(e);
      }
    });
  }

  @Override
  protected void serviceStop() throws Exception {
    druidNodes.stream().forEach(node -> {
      try {
        node.close();
      } catch (IOException e) {
        // nothing that we can really do about it
        log.error(String.format("Failed to stop druid node [%s]", node.getNodeType()), e);
      }
    });
  }


  public String getMetadataURI() {
    return derbyURI;
  }

  public String getDeepStorageDir() {
    return dataDirectory.getAbsolutePath() + File.separator + "deep-storage";
  }

  public String getCoordinatorURI(){
    return "localhost:8081";
  }

  public String getOverlordURI(){
    // Overlord and coordinator both run in same JVM.
    return getCoordinatorURI();
  }
}
{'fraction_non_alphanumeric': 0.10893612092430648, 'fraction_numerical': 0.009903283720391499, 'mean_word_length': 2.9281164695177435, 'pattern_counts': {'":': 5, '<': 25, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '421302', 'n_tokens_mistral': 5144, 'n_tokens_neox': 5005, 'n_words': 1231}