<commit_before>package com.sequenceiq.cloudbreak.service.cluster;

import static com.sequenceiq.cloudbreak.service.stack.connector.VolumeUtils.buildVolumePathString;
import static com.sequenceiq.cloudbreak.service.stack.connector.VolumeUtils.getLogVolume;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import javax.annotation.PostConstruct;
import javax.inject.Inject;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import com.fasterxml.jackson.databind.JsonNode;
import com.sequenceiq.cloudbreak.domain.Cluster;
import com.sequenceiq.cloudbreak.domain.HostGroup;
import com.sequenceiq.cloudbreak.repository.HostGroupRepository;
import com.sequenceiq.cloudbreak.service.CloudbreakServiceException;
import com.sequenceiq.cloudbreak.util.FileReaderUtils;
import com.sequenceiq.cloudbreak.util.JsonUtil;

@Service
public class HadoopConfigurationService {

    @Inject
    private HostGroupRepository hostGroupRepository;

    private Map<String, ServiceConfig> serviceConfigs = new HashMap<>();

    private Map<String, Map<String, String>> bpConfigs = new HashMap<>();

    @Value("#{'${cb.byos.dfs.data.dir}'.split('\\,')}")
    private List<String> byosDfsDataDirs;

    @PostConstruct
    public void init() throws IOException {
        String serviceConfigJson = FileReaderUtils.readFileFromClasspath("hdp/service-config.json");
        JsonNode services = JsonUtil.readTree(serviceConfigJson).get("services");
        for (JsonNode service : services) {
            String serviceName = service.get("name").asText();
            JsonNode configurations = service.get("configurations");
            Map<String, List<ConfigProperty>> globalConfig = new HashMap<>();
            Map<String, List<ConfigProperty>> hostConfig = new HashMap<>();
            for (JsonNode config : configurations) {
                String type = config.get("type").asText();
                List<ConfigProperty> global = toList(config.get("global"));
                if (!global.isEmpty()) {
                    globalConfig.put(type, global);
                }
                List<ConfigProperty> host = toList(config.get("host"));
                if (!host.isEmpty()) {
                    hostConfig.put(type, host);
                }
            }
            serviceConfigs.put(serviceName, new ServiceConfig(serviceName, globalConfig, hostConfig));
        }

        String bpConfigJson = FileReaderUtils.readFileFromClasspath("hdp/bp-config.json");
        JsonNode bps = JsonUtil.readTree(bpConfigJson).get("sites");
        for (JsonNode bp : bps) {
            String siteName = bp.get("name").asText();
            JsonNode configurations = bp.get("configurations");
            Map<String, String> keyVals = new HashMap<>();
            for (JsonNode config : configurations) {
                String key = config.get("key").asText();
                String value = config.get("value").asText();
                keyVals.put(key, value);
            }
            bpConfigs.put(siteName, keyVals);
        }
    }

    public Map<String, Map<String, String>> getGlobalConfiguration(Cluster cluster) throws IOException {
        Set<HostGroup> hostGroups = hostGroupRepository.findHostGroupsInCluster(cluster.getId());
        Map<String, Map<String, String>> config = new HashMap<>();
        JsonNode blueprintNode = JsonUtil.readTree(cluster.getBlueprint().getBlueprintText());
        JsonNode hostGroupsBp = blueprintNode.path("host_groups");
        for (JsonNode hostGroupNode : hostGroupsBp) {
            HostGroup hostGroup = findHostGroupForNode(hostGroups, hostGroupNode);
            JsonNode components = hostGroupNode.path("components");
            for (JsonNode component : components) {
                String name = component.path("name").asText();
                Integer volumeCount = -1;
                if (hostGroup.getConstraint().getInstanceGroup() != null) {
                    volumeCount = null;
                }
                config.putAll(getProperties(name, true, volumeCount));
            }
        }
        for (Map.Entry<String, Map<String, String>> entry : bpConfigs.entrySet()) {
            if (config.containsKey(entry.getKey())) {
                for (Map.Entry<String, String> inEntry : entry.getValue().entrySet()) {
                    config.get(entry.getKey()).put(inEntry.getKey(), inEntry.getValue());
                }
            } else {
                config.put(entry.getKey(), entry.getValue());
            }
        }
        return config;
    }

    private HostGroup findHostGroupForNode(Set<HostGroup> hostGroups, JsonNode hostGroupNode) {
        for (HostGroup hostGroup : hostGroups) {
            if (hostGroup.getName().equals(hostGroupNode.path("name").asText())) {
                return hostGroup;
            }
        }
        throw new CloudbreakServiceException("Couldn't find a saved hostgroup for the hostgroup in the blueprint.");
    }

    public Map<String, Map<String, Map<String, String>>> getHostGroupConfiguration(Cluster cluster) {
        Set<HostGroup> hostGroups = hostGroupRepository.findHostGroupsInCluster(cluster.getId());
        Map<String, Map<String, Map<String, String>>> hadoopConfig = new HashMap<>();
        for (HostGroup hostGroup : hostGroups) {
            Map<String, Map<String, String>> componentConfig = new HashMap<>();
            Integer volumeCount = -1;
            if (hostGroup.getConstraint().getInstanceGroup() != null) {
                volumeCount = hostGroup.getConstraint().getInstanceGroup().getTemplate().getVolumeCount();
            }
            if (configUpdateNeeded(hostGroup)) {
                for (String serviceName : serviceConfigs.keySet()) {<commit_msg>CLOUD-79673 Null check instance group templates for orchestrators<commit_after>package com.sequenceiq.cloudbreak.service.cluster;

import static com.sequenceiq.cloudbreak.service.stack.connector.VolumeUtils.buildVolumePathString;
import static com.sequenceiq.cloudbreak.service.stack.connector.VolumeUtils.getLogVolume;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import javax.annotation.PostConstruct;
import javax.inject.Inject;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import com.fasterxml.jackson.databind.JsonNode;
import com.sequenceiq.cloudbreak.domain.Cluster;
import com.sequenceiq.cloudbreak.domain.HostGroup;
import com.sequenceiq.cloudbreak.repository.HostGroupRepository;
import com.sequenceiq.cloudbreak.service.CloudbreakServiceException;
import com.sequenceiq.cloudbreak.util.FileReaderUtils;
import com.sequenceiq.cloudbreak.util.JsonUtil;

@Service
public class HadoopConfigurationService {

    @Inject
    private HostGroupRepository hostGroupRepository;

    private Map<String, ServiceConfig> serviceConfigs = new HashMap<>();

    private Map<String, Map<String, String>> bpConfigs = new HashMap<>();

    @Value("#{'${cb.byos.dfs.data.dir}'.split('\\,')}")
    private List<String> byosDfsDataDirs;

    @PostConstruct
    public void init() throws IOException {
        String serviceConfigJson = FileReaderUtils.readFileFromClasspath("hdp/service-config.json");
        JsonNode services = JsonUtil.readTree(serviceConfigJson).get("services");
        for (JsonNode service : services) {
            String serviceName = service.get("name").asText();
            JsonNode configurations = service.get("configurations");
            Map<String, List<ConfigProperty>> globalConfig = new HashMap<>();
            Map<String, List<ConfigProperty>> hostConfig = new HashMap<>();
            for (JsonNode config : configurations) {
                String type = config.get("type").asText();
                List<ConfigProperty> global = toList(config.get("global"));
                if (!global.isEmpty()) {
                    globalConfig.put(type, global);
                }
                List<ConfigProperty> host = toList(config.get("host"));
                if (!host.isEmpty()) {
                    hostConfig.put(type, host);
                }
            }
            serviceConfigs.put(serviceName, new ServiceConfig(serviceName, globalConfig, hostConfig));
        }

        String bpConfigJson = FileReaderUtils.readFileFromClasspath("hdp/bp-config.json");
        JsonNode bps = JsonUtil.readTree(bpConfigJson).get("sites");
        for (JsonNode bp : bps) {
            String siteName = bp.get("name").asText();
            JsonNode configurations = bp.get("configurations");
            Map<String, String> keyVals = new HashMap<>();
            for (JsonNode config : configurations) {
                String key = config.get("key").asText();
                String value = config.get("value").asText();
                keyVals.put(key, value);
            }
            bpConfigs.put(siteName, keyVals);
        }
    }

    public Map<String, Map<String, String>> getGlobalConfiguration(Cluster cluster) throws IOException {
        Set<HostGroup> hostGroups = hostGroupRepository.findHostGroupsInCluster(cluster.getId());
        Map<String, Map<String, String>> config = new HashMap<>();
        JsonNode blueprintNode = JsonUtil.readTree(cluster.getBlueprint().getBlueprintText());
        JsonNode hostGroupsBp = blueprintNode.path("host_groups");
        for (JsonNode hostGroupNode : hostGroupsBp) {
            HostGroup hostGroup = findHostGroupForNode(hostGroups, hostGroupNode);
            JsonNode components = hostGroupNode.path("components");
            for (JsonNode component : components) {
                String name = component.path("name").asText();
                Integer volumeCount = -1;
                if (hostGroup.getConstraint().getInstanceGroup() != null) {
                    volumeCount = null;
                }
                config.putAll(getProperties(name, true, volumeCount));
            }
        }
        for (Map.Entry<String, Map<String, String>> entry : bpConfigs.entrySet()) {
            if (config.containsKey(entry.getKey())) {
                for (Map.Entry<String, String> inEntry : entry.getValue().entrySet()) {
                    config.get(entry.getKey()).put(inEntry.getKey(), inEntry.getValue());
                }
            } else {
                config.put(entry.getKey(), entry.getValue());
            }
        }
        return config;
    }

    private HostGroup findHostGroupForNode(Set<HostGroup> hostGroups, JsonNode hostGroupNode) {
        for (HostGroup hostGroup : hostGroups) {
            if (hostGroup.getName().equals(hostGroupNode.path("name").asText())) {
                return hostGroup;
            }
        }
        throw new CloudbreakServiceException("Couldn't find a saved hostgroup for the hostgroup in the blueprint.");
    }

    public Map<String, Map<String, Map<String, String>>> getHostGroupConfiguration(Cluster cluster) {
        Set<HostGroup> hostGroups = hostGroupRepository.findHostGroupsInCluster(cluster.getId());
        Map<String, Map<String, Map<String, String>>> hadoopConfig = new HashMap<>();
        for (HostGroup hostGroup : hostGroups) {
            Map<String, Map<String, String>> componentConfig = new HashMap<>();
            Integer volumeCount = -1;
            if (hostGroup.getConstraint().getInstanceGroup().getTemplate() != null) {
                volumeCount = hostGroup.getConstraint().getInstanceGroup().getTemplate().getVolumeCount();
            }
            if (configUpdateNeeded(hostGroup)) {
                for (String serviceName : serviceConfigs.keySet()) {
{'fraction_non_alphanumeric': 0.11635807711907593, 'fraction_numerical': 0.0007643961270596229, 'mean_word_length': 2.823051948051948, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '424246', 'n_tokens_mistral': 3062, 'n_tokens_neox': 2978, 'n_words': 607}