<commit_before>/*
 * Druid - a distributed column store.
 * Copyright (C) 2012  Metamarkets Group Inc.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

package com.metamx.druid.master;

import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.metamx.common.ISE;
import com.metamx.common.Pair;
import com.metamx.common.concurrent.ScheduledExecutorFactory;
import com.metamx.common.concurrent.ScheduledExecutors;
import com.metamx.common.guava.Comparators;
import com.metamx.common.guava.FunctionalIterable;
import com.metamx.common.lifecycle.LifecycleStart;
import com.metamx.common.lifecycle.LifecycleStop;
import com.metamx.common.logger.Logger;
import com.metamx.druid.client.DataSegment;
import com.metamx.druid.client.DruidDataSource;
import com.metamx.druid.client.DruidServer;
import com.metamx.druid.client.ServerInventoryManager;
import com.metamx.druid.coordination.DruidClusterInfo;
import com.metamx.druid.db.DatabaseRuleManager;
import com.metamx.druid.db.DatabaseSegmentManager;
import com.metamx.emitter.EmittingLogger;
import com.metamx.emitter.service.ServiceEmitter;
import com.metamx.emitter.service.ServiceMetricEvent;
import com.metamx.phonebook.PhoneBook;
import com.metamx.phonebook.PhoneBookPeon;
import com.netflix.curator.x.discovery.ServiceProvider;
import org.I0Itec.zkclient.exception.ZkNodeExistsException;
import org.codehaus.jackson.map.ObjectMapper;
import org.joda.time.DateTime;
import org.joda.time.Duration;

import javax.annotation.Nullable;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ScheduledExecutorService;

/**
 */
public class DruidMaster
{
  public static final String MASTER_OWNER_NODE = "_MASTER";

  private static final EmittingLogger log = new EmittingLogger(DruidMaster.class);

  private final Object lock = new Object();

  private volatile boolean started = false;
  private volatile boolean master = false;

  private final DruidMasterConfig config;
  private final DruidClusterInfo clusterInfo;
  private final DatabaseSegmentManager databaseSegmentManager;
  private final ServerInventoryManager serverInventoryManager;
  private final DatabaseRuleManager databaseRuleManager;
  private final PhoneBook yp;
  private final ServiceEmitter emitter;
  private final ScheduledExecutorService exec;
  private final ScheduledExecutorService peonExec;
  private final PhoneBookPeon masterPeon;
  private final Map<String, LoadQueuePeon> loadManagementPeons;
  private final ServiceProvider serviceProvider;

  private final ObjectMapper jsonMapper;

  public DruidMaster(
      DruidMasterConfig config,
      DruidClusterInfo clusterInfo,
      ObjectMapper jsonMapper,
      DatabaseSegmentManager databaseSegmentManager,
      ServerInventoryManager serverInventoryManager,
      DatabaseRuleManager databaseRuleManager,
      PhoneBook zkPhoneBook,
      ServiceEmitter emitter,
      ScheduledExecutorFactory scheduledExecutorFactory,
      Map<String, LoadQueuePeon> loadManagementPeons,
      ServiceProvider serviceProvider
  )
  {
    this.config = config;
    this.clusterInfo = clusterInfo;

    this.jsonMapper = jsonMapper;

    this.databaseSegmentManager = databaseSegmentManager;
    this.serverInventoryManager = serverInventoryManager;
    this.databaseRuleManager = databaseRuleManager;
    this.yp = zkPhoneBook;
    this.emitter = emitter;

    this.masterPeon = new MasterListeningPeon();
    this.exec = scheduledExecutorFactory.create(1, "Master-Exec--%d");
    this.peonExec = scheduledExecutorFactory.create(1, "Master-PeonExec--%d");

    this.loadManagementPeons = loadManagementPeons;

    this.serviceProvider = serviceProvider;
  }

  public boolean isClusterMaster()
  {
    return master;
  }

  public DruidMasterConfig getConfig()
  {
    return config;
  }

  public Map<String, Double> getLoadStatus()
  {
    Map<String, Integer> availableSegmentMap = Maps.newHashMap();

    for (DataSegment segment : getAvailableDataSegments()) {
      Integer count = availableSegmentMap.get(segment.getDataSource());
      int newCount = (count == null) ? 0 : count.intValue();
      availableSegmentMap.put(segment.getDataSource(), ++newCount);
    }

    Map<String, Set<DataSegment>> loadedDataSources = Maps.newHashMap();
    for (DruidServer server : serverInventoryManager.getInventory()) {
      for (DruidDataSource dataSource : server.getDataSources()) {
        if (!loadedDataSources.containsKey(dataSource.getName())) {
          TreeSet<DataSegment> setToAdd = Sets.newTreeSet(DataSegment.bucketMonthComparator());
          setToAdd.addAll(dataSource.getSegments());
          loadedDataSources.put(dataSource.getName(), setToAdd);
        } else {
          loadedDataSources.get(dataSource.getName()).addAll(dataSource.getSegments());
        }
      }
    }

    Map<String, Integer> loadedSegmentMap = Maps.newHashMap();<commit_msg>remove getConfig method from DruidMaster<commit_after>/*
 * Druid - a distributed column store.
 * Copyright (C) 2012  Metamarkets Group Inc.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

package com.metamx.druid.master;

import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.metamx.common.ISE;
import com.metamx.common.Pair;
import com.metamx.common.concurrent.ScheduledExecutorFactory;
import com.metamx.common.concurrent.ScheduledExecutors;
import com.metamx.common.guava.Comparators;
import com.metamx.common.guava.FunctionalIterable;
import com.metamx.common.lifecycle.LifecycleStart;
import com.metamx.common.lifecycle.LifecycleStop;
import com.metamx.common.logger.Logger;
import com.metamx.druid.client.DataSegment;
import com.metamx.druid.client.DruidDataSource;
import com.metamx.druid.client.DruidServer;
import com.metamx.druid.client.ServerInventoryManager;
import com.metamx.druid.coordination.DruidClusterInfo;
import com.metamx.druid.db.DatabaseRuleManager;
import com.metamx.druid.db.DatabaseSegmentManager;
import com.metamx.emitter.EmittingLogger;
import com.metamx.emitter.service.ServiceEmitter;
import com.metamx.emitter.service.ServiceMetricEvent;
import com.metamx.phonebook.PhoneBook;
import com.metamx.phonebook.PhoneBookPeon;
import com.netflix.curator.x.discovery.ServiceProvider;
import org.I0Itec.zkclient.exception.ZkNodeExistsException;
import org.codehaus.jackson.map.ObjectMapper;
import org.joda.time.DateTime;
import org.joda.time.Duration;

import javax.annotation.Nullable;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ScheduledExecutorService;

/**
 */
public class DruidMaster
{
  public static final String MASTER_OWNER_NODE = "_MASTER";

  private static final EmittingLogger log = new EmittingLogger(DruidMaster.class);

  private final Object lock = new Object();

  private volatile boolean started = false;
  private volatile boolean master = false;

  private final DruidMasterConfig config;
  private final DruidClusterInfo clusterInfo;
  private final DatabaseSegmentManager databaseSegmentManager;
  private final ServerInventoryManager serverInventoryManager;
  private final DatabaseRuleManager databaseRuleManager;
  private final PhoneBook yp;
  private final ServiceEmitter emitter;
  private final ScheduledExecutorService exec;
  private final ScheduledExecutorService peonExec;
  private final PhoneBookPeon masterPeon;
  private final Map<String, LoadQueuePeon> loadManagementPeons;
  private final ServiceProvider serviceProvider;

  private final ObjectMapper jsonMapper;

  public DruidMaster(
      DruidMasterConfig config,
      DruidClusterInfo clusterInfo,
      ObjectMapper jsonMapper,
      DatabaseSegmentManager databaseSegmentManager,
      ServerInventoryManager serverInventoryManager,
      DatabaseRuleManager databaseRuleManager,
      PhoneBook zkPhoneBook,
      ServiceEmitter emitter,
      ScheduledExecutorFactory scheduledExecutorFactory,
      Map<String, LoadQueuePeon> loadManagementPeons,
      ServiceProvider serviceProvider
  )
  {
    this.config = config;
    this.clusterInfo = clusterInfo;

    this.jsonMapper = jsonMapper;

    this.databaseSegmentManager = databaseSegmentManager;
    this.serverInventoryManager = serverInventoryManager;
    this.databaseRuleManager = databaseRuleManager;
    this.yp = zkPhoneBook;
    this.emitter = emitter;

    this.masterPeon = new MasterListeningPeon();
    this.exec = scheduledExecutorFactory.create(1, "Master-Exec--%d");
    this.peonExec = scheduledExecutorFactory.create(1, "Master-PeonExec--%d");

    this.loadManagementPeons = loadManagementPeons;

    this.serviceProvider = serviceProvider;
  }

  public boolean isClusterMaster()
  {
    return master;
  }

  public Map<String, Double> getLoadStatus()
  {
    Map<String, Integer> availableSegmentMap = Maps.newHashMap();

    for (DataSegment segment : getAvailableDataSegments()) {
      Integer count = availableSegmentMap.get(segment.getDataSource());
      int newCount = (count == null) ? 0 : count.intValue();
      availableSegmentMap.put(segment.getDataSource(), ++newCount);
    }

    Map<String, Set<DataSegment>> loadedDataSources = Maps.newHashMap();
    for (DruidServer server : serverInventoryManager.getInventory()) {
      for (DruidDataSource dataSource : server.getDataSources()) {
        if (!loadedDataSources.containsKey(dataSource.getName())) {
          TreeSet<DataSegment> setToAdd = Sets.newTreeSet(DataSegment.bucketMonthComparator());
          setToAdd.addAll(dataSource.getSegments());
          loadedDataSources.put(dataSource.getName(), setToAdd);
        } else {
          loadedDataSources.get(dataSource.getName()).addAll(dataSource.getSegments());
        }
      }
    }

    Map<String, Integer> loadedSegmentMap = Maps.newHashMap();
{'fraction_non_alphanumeric': 0.08536381842456608, 'fraction_numerical': 0.0035046728971962616, 'mean_word_length': 4.764790764790765, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4668547', 'n_tokens_mistral': 3416, 'n_tokens_neox': 3362, 'n_words': 824}