<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package io.brooklyn.ambari;

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.transform;

import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.LoggerFactory;

import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;

import brooklyn.config.ConfigKey;
import brooklyn.enricher.Enrichers;
import brooklyn.entity.basic.Attributes;
import brooklyn.entity.basic.BasicStartableImpl;
import brooklyn.entity.basic.BrooklynTaskTags;
import brooklyn.entity.basic.Entities;
import brooklyn.entity.basic.EntityLocal;
import brooklyn.entity.basic.ServiceStateLogic;
import brooklyn.entity.basic.SoftwareProcess;
import brooklyn.entity.proxying.EntitySpec;
import brooklyn.event.SensorEvent;
import brooklyn.event.SensorEventListener;
import brooklyn.location.Location;
import brooklyn.management.Task;
import brooklyn.util.collections.MutableList;
import brooklyn.util.collections.MutableMap;
import brooklyn.util.task.Tasks;
import io.brooklyn.ambari.agent.AmbariAgent;
import io.brooklyn.ambari.agent.AmbariAgentImpl;
import io.brooklyn.ambari.hostgroup.AmbariHostGroup;
import io.brooklyn.ambari.rest.AmbariApiException;
import io.brooklyn.ambari.rest.domain.Bindings;
import io.brooklyn.ambari.rest.domain.Blueprint;
import io.brooklyn.ambari.rest.domain.HostComponent;
import io.brooklyn.ambari.rest.domain.HostGroup;
import io.brooklyn.ambari.rest.domain.Recommendation;
import io.brooklyn.ambari.rest.domain.RecommendationWrapper;
import io.brooklyn.ambari.rest.domain.RecommendationWrappers;
import io.brooklyn.ambari.rest.domain.Request;
import io.brooklyn.ambari.rest.domain.Stack;
import io.brooklyn.ambari.server.AmbariServer;
import io.brooklyn.ambari.service.ExtraService;
import io.brooklyn.ambari.service.ExtraServiceException;

/**
 * The minimum requirements for an ambari hadoop cluster.
 * These can be set in the provisioning properties of yaml or by
 * using machines of this spec in a byon cluster.
 * <p/>
 * "minRam", 8192,
 * "osFamily", "ubuntu",
 * "osVersionRegex", "12.*",
 */
public class AmbariClusterImpl extends BasicStartableImpl implements AmbariCluster {

    private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(BasicStartableImpl.class);
    public static final ImmutableList<String> DEFAULT_SERVICES = ImmutableList.<String>of("ZOOKEEPER");
    public static final ImmutableMap<String, Map> DEFAULT_CONFIG_MAP = ImmutableMap.<String, Map>of();
    //TODO is there an issue with rebind here?  On rebind should be populated from somewhere else?

    private boolean isHostGroupsDeployment;
    private List<String> services;
    private Map<String, Map> configuration;
    private Map<String, List<String>> componentsByNode;
    private Function<AmbariServer, String> mapAmbariServerToFQDN = new Function<AmbariServer, String>() {
        @Nullable
        @Override
        public String apply(@Nullable AmbariServer ambariServer) {
            return ambariServer.getFqdn();
        }
    };

    @Override
    public void init() {
        super.init();

        isHostGroupsDeployment = Iterables.size(getHostGroups()) > 0;

        addChild(createServerSpec(getConfig(SECURITY_GROUP)));
        if (!getConfig(SERVER_COMPONENTS).isEmpty()) {
            for (AmbariServer ambariServer : getAmbariServers()) {
                ambariServer.config().set(SoftwareProcess.CHILDREN_STARTABLE_MODE, SoftwareProcess.ChildStartableMode.BACKGROUND_LATE);
                EntitySpec<? extends AmbariAgent> agentSpec = AmbariAgentImpl.createAgentSpec(this, null);
                ambariServer.addChild(agentSpec);
            }
        }

        configuration = MutableMap.copyOf(getConfig(AMBARI_CONFIGURATIONS));
        services = MutableList.copyOf(getConfig(HADOOP_SERVICES));

        calculateTotalAgents();
        if (!isHostGroupsDeployment) {
            createClusterTopology();
            if (services.size() == 0) {
                services.addAll(DEFAULT_SERVICES);
            }
        }

        if (configuration.size() == 0) {
            configuration.putAll(DEFAULT_CONFIG_MAP);
        }

        addEnricher(Enrichers.builder()
                .propagating(Attributes.MAIN_URI)
                .from(getMasterAmbariServer())
                .build());

        componentsByNode = new MutableMap<String, List<String>>();
        addDeprecatedExtraServiceToExtraServices();
        for (EntitySpec<? extends ExtraService> entitySpec : getConfig(EXTRA_HADOOP_SERVICES)) {
            LOG.warn(EXTRA_HADOOP_SERVICES.getName() + " configuration key is deprecated. Extra services should now be defined through as children by using 'brooklyn.children'");
            addChild(entitySpec);
        }

        final Iterable<String> ambariHostGroupNames = transform(getHostGroups(), new Function<AmbariHostGroup, String>() {
            @Nullable
            @Override
            public String apply(@Nullable AmbariHostGroup ambariHostGroup) {
                return ambariHostGroup != null ? ambariHostGroup.getDisplayName() : null;
            }
        });

        for (ExtraService extraService : Entities.descendants(this, ExtraService.class)) {
            if (isHostGroupsDeployment) {
                Preconditions.checkNotNull(extraService.getConfig(ExtraService.COMPONENT_NAMES),
                        "Entity \"%s\" must define a list of components names as this is a host groups based deployment. Please use the \"%s\" configuration key",
                        extraService.getEntityType().getName(),
                        ExtraService.COMPONENT_NAMES.getName());

                for (ExtraService.ComponentMapping componentMapping : extraService.getComponentMappings()) {
                    if (!Iterables.contains(ambariHostGroupNames, componentMapping.getHost())) {
                        throw new IllegalStateException(String.format("Extra component \"%s\" of entity \"%s\" cannot be bound to \"%s\" host group because it does not exist. Please choose from %s",
                                componentMapping.getComponent(), extraService.getEntityType().getName(), componentMapping.getHost(), ambariHostGroupNames));
                    }
                    if (!componentsByNode.containsKey(componentMapping.getHost())) {
                        componentsByNode.put(componentMapping.getHost(), MutableList.<String>of());
                    }
                    componentsByNode.get(componentMapping.getHost()).add(componentMapping.getComponent());
                }
            } else {
                Preconditions.checkNotNull(extraService.getConfig(ExtraService.SERVICE_NAME),
                        "Entity \"%s\" must define a service name as this is a services based deployment. Please use the \"%s\" configuration key",
                        extraService.getEntityType().getName(),
                        ExtraService.SERVICE_NAME.getName());

                if (StringUtils.isNotBlank(extraService.getConfig(ExtraService.SERVICE_NAME))) {
                    services.add(extraService.getConfig(ExtraService.SERVICE_NAME));
                }
            }
            if (extraService.getAmbariConfig() != null) {
                configuration.putAll(extraService.getAmbariConfig());
            }<commit_msg>Allow components to map to server-group<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package io.brooklyn.ambari;

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.transform;

import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.LoggerFactory;

import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;

import brooklyn.config.ConfigKey;
import brooklyn.enricher.Enrichers;
import brooklyn.entity.basic.Attributes;
import brooklyn.entity.basic.BasicStartableImpl;
import brooklyn.entity.basic.BrooklynTaskTags;
import brooklyn.entity.basic.Entities;
import brooklyn.entity.basic.EntityLocal;
import brooklyn.entity.basic.ServiceStateLogic;
import brooklyn.entity.basic.SoftwareProcess;
import brooklyn.entity.proxying.EntitySpec;
import brooklyn.event.SensorEvent;
import brooklyn.event.SensorEventListener;
import brooklyn.location.Location;
import brooklyn.management.Task;
import brooklyn.util.collections.MutableList;
import brooklyn.util.collections.MutableMap;
import brooklyn.util.task.Tasks;
import io.brooklyn.ambari.agent.AmbariAgent;
import io.brooklyn.ambari.agent.AmbariAgentImpl;
import io.brooklyn.ambari.hostgroup.AmbariHostGroup;
import io.brooklyn.ambari.rest.AmbariApiException;
import io.brooklyn.ambari.rest.domain.Bindings;
import io.brooklyn.ambari.rest.domain.Blueprint;
import io.brooklyn.ambari.rest.domain.HostComponent;
import io.brooklyn.ambari.rest.domain.HostGroup;
import io.brooklyn.ambari.rest.domain.Recommendation;
import io.brooklyn.ambari.rest.domain.RecommendationWrapper;
import io.brooklyn.ambari.rest.domain.RecommendationWrappers;
import io.brooklyn.ambari.rest.domain.Request;
import io.brooklyn.ambari.rest.domain.Stack;
import io.brooklyn.ambari.server.AmbariServer;
import io.brooklyn.ambari.service.ExtraService;
import io.brooklyn.ambari.service.ExtraServiceException;

/**
 * The minimum requirements for an ambari hadoop cluster.
 * These can be set in the provisioning properties of yaml or by
 * using machines of this spec in a byon cluster.
 * <p/>
 * "minRam", 8192,
 * "osFamily", "ubuntu",
 * "osVersionRegex", "12.*",
 */
public class AmbariClusterImpl extends BasicStartableImpl implements AmbariCluster {

    private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(BasicStartableImpl.class);
    public static final ImmutableList<String> DEFAULT_SERVICES = ImmutableList.<String>of("ZOOKEEPER");
    public static final ImmutableMap<String, Map> DEFAULT_CONFIG_MAP = ImmutableMap.<String, Map>of();
    //TODO is there an issue with rebind here?  On rebind should be populated from somewhere else?

    private boolean isHostGroupsDeployment;
    private List<String> services;
    private Map<String, Map> configuration;
    private Map<String, List<String>> componentsByNode;
    private Function<AmbariServer, String> mapAmbariServerToFQDN = new Function<AmbariServer, String>() {
        @Nullable
        @Override
        public String apply(@Nullable AmbariServer ambariServer) {
            return ambariServer.getFqdn();
        }
    };

    @Override
    public void init() {
        super.init();

        isHostGroupsDeployment = Iterables.size(getHostGroups()) > 0;

        addChild(createServerSpec(getConfig(SECURITY_GROUP)));
        if (!getConfig(SERVER_COMPONENTS).isEmpty()) {
            for (AmbariServer ambariServer : getAmbariServers()) {
                ambariServer.config().set(SoftwareProcess.CHILDREN_STARTABLE_MODE, SoftwareProcess.ChildStartableMode.BACKGROUND_LATE);
                EntitySpec<? extends AmbariAgent> agentSpec = AmbariAgentImpl.createAgentSpec(this, null);
                ambariServer.addChild(agentSpec);
            }
        }

        configuration = MutableMap.copyOf(getConfig(AMBARI_CONFIGURATIONS));
        services = MutableList.copyOf(getConfig(HADOOP_SERVICES));

        calculateTotalAgents();
        if (!isHostGroupsDeployment) {
            createClusterTopology();
            if (services.size() == 0) {
                services.addAll(DEFAULT_SERVICES);
            }
        }

        if (configuration.size() == 0) {
            configuration.putAll(DEFAULT_CONFIG_MAP);
        }

        addEnricher(Enrichers.builder()
                .propagating(Attributes.MAIN_URI)
                .from(getMasterAmbariServer())
                .build());

        componentsByNode = new MutableMap<String, List<String>>();
        addDeprecatedExtraServiceToExtraServices();
        for (EntitySpec<? extends ExtraService> entitySpec : getConfig(EXTRA_HADOOP_SERVICES)) {
            LOG.warn(EXTRA_HADOOP_SERVICES.getName() + " configuration key is deprecated. Extra services should now be defined through as children by using 'brooklyn.children'");
            addChild(entitySpec);
        }

        final Iterable<String> ambariHostGroupNames = transform(getHostGroups(), new Function<AmbariHostGroup, String>() {
            @Nullable
            @Override
            public String apply(@Nullable AmbariHostGroup ambariHostGroup) {
                return ambariHostGroup != null ? ambariHostGroup.getDisplayName() : null;
            }
        });

        for (ExtraService extraService : Entities.descendants(this, ExtraService.class)) {
            if (isHostGroupsDeployment) {
                Preconditions.checkNotNull(extraService.getConfig(ExtraService.COMPONENT_NAMES),
                        "Entity \"%s\" must define a list of components names as this is a host groups based deployment. Please use the \"%s\" configuration key",
                        extraService.getEntityType().getName(),
                        ExtraService.COMPONENT_NAMES.getName());

                for (ExtraService.ComponentMapping componentMapping : extraService.getComponentMappings()) {
                    if (!componentMapping.getHost().equals(SERVER_HOST_GROUP) && !Iterables.contains(ambariHostGroupNames, componentMapping.getHost())) {
                        throw new IllegalStateException(String.format("Extra component \"%s\" of entity \"%s\" cannot be bound to \"%s\" host group because it does not exist. Please choose from %s or " + SERVER_HOST_GROUP,
                                componentMapping.getComponent(), extraService.getEntityType().getName(), componentMapping.getHost(), ambariHostGroupNames));
                    }
                    if (!componentsByNode.containsKey(componentMapping.getHost())) {
                        componentsByNode.put(componentMapping.getHost(), MutableList.<String>of());
                    }
                    componentsByNode.get(componentMapping.getHost()).add(componentMapping.getComponent());
                }
            } else {
                Preconditions.checkNotNull(extraService.getConfig(ExtraService.SERVICE_NAME),
                        "Entity \"%s\" must define a service name as this is a services based deployment. Please use the \"%s\" configuration key",
                        extraService.getEntityType().getName(),
                        ExtraService.SERVICE_NAME.getName());

                if (StringUtils.isNotBlank(extraService.getConfig(ExtraService.SERVICE_NAME))) {
                    services.add(extraService.getConfig(ExtraService.SERVICE_NAME));
                }
            }
            if (extraService.getAmbariConfig() != null) {
                configuration.putAll(extraService.getAmbariConfig());
            }
{'fraction_non_alphanumeric': 0.09427394558217787, 'fraction_numerical': 0.0019725010152578755, 'mean_word_length': 3.2976813762154076, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2013461', 'n_tokens_mistral': 4747, 'n_tokens_neox': 4519, 'n_words': 1054}