<commit_before>package com.thinkbiganalytics.nifi.feedmgr;

import com.google.common.collect.Lists;
import com.thinkbiganalytics.nifi.rest.client.LegacyNifiRestClient;
import com.thinkbiganalytics.nifi.rest.model.NifiError;
import com.thinkbiganalytics.nifi.rest.model.NifiProcessGroup;
import com.thinkbiganalytics.nifi.rest.model.NifiProperty;
import com.thinkbiganalytics.nifi.rest.support.NifiProcessUtil;
import com.thinkbiganalytics.nifi.rest.support.NifiPropertyUtil;

import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.web.api.dto.ProcessGroupDTO;
import org.apache.nifi.web.api.dto.ProcessorDTO;
import org.apache.nifi.web.api.dto.TemplateDTO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;

/**
 * Created by sr186054 on 5/6/16.
 */
public class TemplateInstanceCreator {

    private static final Logger log = LoggerFactory.getLogger(TemplateInstanceCreator.class);

    private String templateId;
    private LegacyNifiRestClient restClient;

    private boolean createReusableFlow;

    private Map<String, Object> staticConfigPropertyMap;

    private Map<String, String> staticConfigPropertyStringMap;

    public TemplateInstanceCreator(LegacyNifiRestClient restClient, String templateId, Map<String, Object> staticConfigPropertyMap, boolean createReusableFlow) {
        this.restClient = restClient;
        this.templateId = templateId;
        this.createReusableFlow = createReusableFlow;
        this.staticConfigPropertyMap = staticConfigPropertyMap;
        if (staticConfigPropertyMap != null) {
            //transform the object map to the String map
            staticConfigPropertyStringMap = staticConfigPropertyMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue() != null ? e.getValue().toString() : null));
        }
        if (staticConfigPropertyStringMap == null) {
            staticConfigPropertyStringMap = new HashMap<>();
        }
    }

    public boolean isCreateReusableFlow() {
        return createReusableFlow;
    }

    private void ensureInputPortsForReuseableTemplate(String processGroupId) {
        ProcessGroupDTO template = restClient.getProcessGroup(processGroupId, false, false);
        String categoryId = template.getParentGroupId();
        restClient.createReusableTemplateInputPort(categoryId, processGroupId);
    }


    public NifiProcessGroup createTemplate() throws TemplateCreationException {
        try {

            NifiProcessGroup newProcessGroup = null;
            TemplateDTO template = restClient.getTemplateById(templateId);

            if (template != null) {

                TemplateCreationHelper templateCreationHelper = new TemplateCreationHelper(this.restClient);
                String processGroupId = null;

                ProcessGroupDTO group = null;
                if (isCreateReusableFlow()) {
                    log.info("Creating a new Reusable flow instance for template: {} ", template.getName());
                    //1 get/create the parent "reusable_templates" processgroup
                    ProcessGroupDTO reusableParentGroup = restClient.getProcessGroupByName("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
                    if (reusableParentGroup == null) {
                        reusableParentGroup = restClient.createProcessGroup("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
                    }
                    ProcessGroupDTO thisGroup = restClient.getProcessGroupByName(reusableParentGroup.getId(), template.getName());
                    if (thisGroup != null) {
                        //version the group
                        log.info("A previous Process group of with this name {} was found.  Versioning it before continuing", thisGroup.getName());
                        templateCreationHelper.versionProcessGroup(thisGroup);

                    }
                    group = restClient.createProcessGroup(reusableParentGroup.getId(), template.getName());
                } else {
                    String tmpName = template.getName() + "_" + System.currentTimeMillis();
                    group = restClient.createProcessGroup(tmpName);
                    log.info("Creating a temporary process group with name {} for template {} ", tmpName, template.getName());
                }
                processGroupId = group.getId();
                if (StringUtils.isNotBlank(processGroupId)) {
                    //snapshot the existing controller services
                    templateCreationHelper.snapshotControllerServiceReferences();
                    log.info("Successfully Snapshot of controller services");
                    //create the flow from the template
                    templateCreationHelper.instantiateFlowFromTemplate(processGroupId, templateId);
                    log.info("Successfully created the temp flow");

                    if (this.createReusableFlow) {
                        ensureInputPortsForReuseableTemplate(processGroupId);
                        log.info("Reusable flow, input ports created successfully.");
                    }

                    //mark the new services that were created as a result of creating the new flow from the template
                    templateCreationHelper.identifyNewlyCreatedControllerServiceReferences();

                    ProcessGroupDTO entity = restClient.getProcessGroup(processGroupId, true, true);

                    //replace static properties and inject values into the flow

                    List<NifiProperty> processorProperties = NifiPropertyUtil.getProperties(entity, restClient.getPropertyDescriptorTransform());
                    if (processorProperties != null) {
                        boolean didReplace = false;
                        for (NifiProperty property : processorProperties) {
                            boolean replaced = ConfigurationPropertyReplacer.resolveStaticConfigurationProperty(property, staticConfigPropertyMap);
                            if (replaced) {
                                //update the properties that are replaced
                                restClient.updateProcessorProperty(property.getProcessGroupId(), property.getProcessorId(), property);
                                didReplace = true;
                            }
                        }
                        //if we replaced any properties, refetch to get the latest data
                        if (didReplace) {
                            entity = restClient.getProcessGroup(processGroupId, true, true);
                        }
                    }

                    //identify the various processors (first level initial processors)
                    List<ProcessorDTO> inputProcessors = NifiProcessUtil.getInputProcessors(entity);

                    ProcessorDTO input = null;
                    List<ProcessorDTO> nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);

                    //if the input is null attempt to get the first input available on the template
                    if (input == null && inputProcessors != null && !inputProcessors.isEmpty()) {
                        input = inputProcessors.get(0);
                    }

                    log.info("Attempt to update/validate controller services for template.");
                    //update any references to the controller services and try to assign the value to an enabled service if it is not already
                    if (input != null) {
                        log.info("attempt to update controllerservices on {} input processor ", input.getName());
                        templateCreationHelper.updateControllerServiceReferences(Lists.newArrayList(inputProcessors), staticConfigPropertyStringMap);
                    }
                    log.info("attempt to update controllerservices on {} processors ", (nonInputProcessors != null ? nonInputProcessors.size() : 0));
                    templateCreationHelper.updateControllerServiceReferences(nonInputProcessors, staticConfigPropertyStringMap);
                    log.info("Controller service validation complete");
                    //refetch processors for updated errors
                    entity = restClient.getProcessGroup(processGroupId, true, true);
                    nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);

                    ///make the input/output ports in the category group as running
                    if (isCreateReusableFlow()) {
                        log.info("Reusable flow, attempt to mark the connection ports as running.");
                        templateCreationHelper.markConnectionPortsAsRunning(entity);
                        log.info("Reusable flow.  Successfully marked the ports as running.");
                    }

                    newProcessGroup = new NifiProcessGroup(entity, input, nonInputProcessors);

                    if (isCreateReusableFlow()) {
                        log.info("Reusable flow, attempt to mark the Processors as running.");
                        templateCreationHelper.markProcessorsAsRunning(newProcessGroup);
                        log.info("Reusable flow.  Successfully marked the Processors as running.");
                    }

                    templateCreationHelper.cleanupControllerServices();
                    log.info("Controller service cleanup complete");
                    List<NifiError> errors = templateCreationHelper.getErrors();
                    //add any global errors to the object
                    if (errors != null && !errors.isEmpty()) {
                        for (NifiError error : errors) {
                            newProcessGroup.addError(error);
                        }
                    }

                    newProcessGroup.setSuccess(!newProcessGroup.hasFatalErrors());
                    log.info("Finished importing template Errors found.  Success: {}, {} {}", newProcessGroup.isSuccess(), (errors != null ? errors.size() : 0),
                             (errors != null ? " - " + StringUtils.join(errors) : ""));

                    return newProcessGroup;

                }
            }<commit_msg>PC-840 Align Reusable templates on import/creation<commit_after>package com.thinkbiganalytics.nifi.feedmgr;

import com.google.common.collect.Lists;
import com.thinkbiganalytics.nifi.rest.client.LegacyNifiRestClient;
import com.thinkbiganalytics.nifi.rest.client.layout.AlignProcessGroupComponents;
import com.thinkbiganalytics.nifi.rest.model.NifiError;
import com.thinkbiganalytics.nifi.rest.model.NifiProcessGroup;
import com.thinkbiganalytics.nifi.rest.model.NifiProperty;
import com.thinkbiganalytics.nifi.rest.support.NifiProcessUtil;
import com.thinkbiganalytics.nifi.rest.support.NifiPropertyUtil;

import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.web.api.dto.ProcessGroupDTO;
import org.apache.nifi.web.api.dto.ProcessorDTO;
import org.apache.nifi.web.api.dto.TemplateDTO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;

/**
 * Created by sr186054 on 5/6/16.
 */
public class TemplateInstanceCreator {

    private static final Logger log = LoggerFactory.getLogger(TemplateInstanceCreator.class);

    private String templateId;
    private LegacyNifiRestClient restClient;

    private boolean createReusableFlow;

    private Map<String, Object> staticConfigPropertyMap;

    private Map<String, String> staticConfigPropertyStringMap;

    public TemplateInstanceCreator(LegacyNifiRestClient restClient, String templateId, Map<String, Object> staticConfigPropertyMap, boolean createReusableFlow) {
        this.restClient = restClient;
        this.templateId = templateId;
        this.createReusableFlow = createReusableFlow;
        this.staticConfigPropertyMap = staticConfigPropertyMap;
        if (staticConfigPropertyMap != null) {
            //transform the object map to the String map
            staticConfigPropertyStringMap = staticConfigPropertyMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue() != null ? e.getValue().toString() : null));
        }
        if (staticConfigPropertyStringMap == null) {
            staticConfigPropertyStringMap = new HashMap<>();
        }
    }

    public boolean isCreateReusableFlow() {
        return createReusableFlow;
    }

    private void ensureInputPortsForReuseableTemplate(String processGroupId) {
        ProcessGroupDTO template = restClient.getProcessGroup(processGroupId, false, false);
        String categoryId = template.getParentGroupId();
        restClient.createReusableTemplateInputPort(categoryId, processGroupId);
    }


    public NifiProcessGroup createTemplate() throws TemplateCreationException {
        try {

            NifiProcessGroup newProcessGroup = null;
            TemplateDTO template = restClient.getTemplateById(templateId);

            if (template != null) {

                TemplateCreationHelper templateCreationHelper = new TemplateCreationHelper(this.restClient);
                String processGroupId = null;

                ProcessGroupDTO group = null;
                if (isCreateReusableFlow()) {
                    log.info("Creating a new Reusable flow instance for template: {} ", template.getName());
                    //1 get/create the parent "reusable_templates" processgroup
                    ProcessGroupDTO reusableParentGroup = restClient.getProcessGroupByName("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
                    if (reusableParentGroup == null) {
                        reusableParentGroup = restClient.createProcessGroup("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
                    }
                    ProcessGroupDTO thisGroup = restClient.getProcessGroupByName(reusableParentGroup.getId(), template.getName());
                    if (thisGroup != null) {
                        //version the group
                        log.info("A previous Process group of with this name {} was found.  Versioning it before continuing", thisGroup.getName());
                        templateCreationHelper.versionProcessGroup(thisGroup);

                    }
                    group = restClient.createProcessGroup(reusableParentGroup.getId(), template.getName());
                } else {
                    String tmpName = template.getName() + "_" + System.currentTimeMillis();
                    group = restClient.createProcessGroup(tmpName);
                    log.info("Creating a temporary process group with name {} for template {} ", tmpName, template.getName());
                }
                processGroupId = group.getId();
                if (StringUtils.isNotBlank(processGroupId)) {
                    //snapshot the existing controller services
                    templateCreationHelper.snapshotControllerServiceReferences();
                    log.info("Successfully Snapshot of controller services");
                    //create the flow from the template
                    templateCreationHelper.instantiateFlowFromTemplate(processGroupId, templateId);
                    log.info("Successfully created the temp flow");

                    if (this.createReusableFlow) {
                        ensureInputPortsForReuseableTemplate(processGroupId);
                        log.info("Reusable flow, input ports created successfully.");
                    }

                    //mark the new services that were created as a result of creating the new flow from the template
                    templateCreationHelper.identifyNewlyCreatedControllerServiceReferences();

                    ProcessGroupDTO entity = restClient.getProcessGroup(processGroupId, true, true);

                    //replace static properties and inject values into the flow

                    List<NifiProperty> processorProperties = NifiPropertyUtil.getProperties(entity, restClient.getPropertyDescriptorTransform());
                    if (processorProperties != null) {
                        boolean didReplace = false;
                        for (NifiProperty property : processorProperties) {
                            boolean replaced = ConfigurationPropertyReplacer.resolveStaticConfigurationProperty(property, staticConfigPropertyMap);
                            if (replaced) {
                                //update the properties that are replaced
                                restClient.updateProcessorProperty(property.getProcessGroupId(), property.getProcessorId(), property);
                                didReplace = true;
                            }
                        }
                        //if we replaced any properties, refetch to get the latest data
                        if (didReplace) {
                            entity = restClient.getProcessGroup(processGroupId, true, true);
                        }
                    }

                    //identify the various processors (first level initial processors)
                    List<ProcessorDTO> inputProcessors = NifiProcessUtil.getInputProcessors(entity);

                    ProcessorDTO input = null;
                    List<ProcessorDTO> nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);

                    //if the input is null attempt to get the first input available on the template
                    if (input == null && inputProcessors != null && !inputProcessors.isEmpty()) {
                        input = inputProcessors.get(0);
                    }

                    log.info("Attempt to update/validate controller services for template.");
                    //update any references to the controller services and try to assign the value to an enabled service if it is not already
                    if (input != null) {
                        log.info("attempt to update controllerservices on {} input processor ", input.getName());
                        templateCreationHelper.updateControllerServiceReferences(Lists.newArrayList(inputProcessors), staticConfigPropertyStringMap);
                    }
                    log.info("attempt to update controllerservices on {} processors ", (nonInputProcessors != null ? nonInputProcessors.size() : 0));
                    templateCreationHelper.updateControllerServiceReferences(nonInputProcessors, staticConfigPropertyStringMap);
                    log.info("Controller service validation complete");
                    //refetch processors for updated errors
                    entity = restClient.getProcessGroup(processGroupId, true, true);
                    nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);

                    ///make the input/output ports in the category group as running
                    if (isCreateReusableFlow()) {
                        log.info("Reusable flow, attempt to mark the connection ports as running.");
                        templateCreationHelper.markConnectionPortsAsRunning(entity);
                        log.info("Reusable flow.  Successfully marked the ports as running.");
                    }

                    newProcessGroup = new NifiProcessGroup(entity, input, nonInputProcessors);

                    if (isCreateReusableFlow()) {
                        log.info("Reusable flow, attempt to mark the Processors as running.");
                        templateCreationHelper.markProcessorsAsRunning(newProcessGroup);
                        log.info("Reusable flow.  Successfully marked the Processors as running.");
                        //align items
                        AlignProcessGroupComponents alignProcessGroupComponents = new AlignProcessGroupComponents(restClient.getNiFiRestClient(), entity.getParentGroupId());
                        alignProcessGroupComponents.autoLayout();
                    }

                    templateCreationHelper.cleanupControllerServices();
                    log.info("Controller service cleanup complete");
                    List<NifiError> errors = templateCreationHelper.getErrors();
                    //add any global errors to the object
                    if (errors != null && !errors.isEmpty()) {
                        for (NifiError error : errors) {
                            newProcessGroup.addError(error);
                        }
                    }

                    newProcessGroup.setSuccess(!newProcessGroup.hasFatalErrors());
                    log.info("Finished importing template Errors found.  Success: {}, {} {}", newProcessGroup.isSuccess(), (errors != null ? errors.size() : 0),
                             (errors != null ? " - " + StringUtils.join(errors) : ""));

                    return newProcessGroup;

                }
            }
{'fraction_non_alphanumeric': 0.07632508833922262, 'fraction_numerical': 0.0017432273262661955, 'mean_word_length': 2.139940828402367, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '837293', 'n_tokens_mistral': 4909, 'n_tokens_neox': 4590, 'n_words': 1125}