<commit_before>/*******************************************************************************
 *  
 *   Copyright 2015 Walmart, Inc.
 *  
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *  
 *       http://www.apache.org/licenses/LICENSE-2.0
 *  
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 *  
 *******************************************************************************/
package com.oneops.transistor.service;

import com.google.gson.Gson;
import com.oneops.cms.cm.domain.CmsCI;
import com.oneops.cms.cm.domain.CmsCIAttribute;
import com.oneops.cms.cm.domain.CmsCIRelation;
import com.oneops.cms.cm.domain.CmsCIRelationAttribute;
import com.oneops.cms.cm.domain.CmsLink;
import com.oneops.cms.cm.service.CmsCmProcessor;
import com.oneops.cms.dj.domain.CmsRfcAttribute;
import com.oneops.cms.dj.domain.CmsRfcCI;
import com.oneops.cms.dj.domain.CmsRfcRelation;
import com.oneops.cms.dj.domain.RfcHint;
import com.oneops.cms.dj.service.CmsCmRfcMrgProcessor;
import com.oneops.cms.dj.service.CmsRfcProcessor;
import com.oneops.cms.exceptions.CIValidationException;
import com.oneops.cms.exceptions.DJException;
import com.oneops.cms.exceptions.ExceptionConsolidator;
import com.oneops.cms.md.domain.CmsClazz;
import com.oneops.cms.md.domain.CmsClazzAttribute;
import com.oneops.cms.md.domain.CmsRelation;
import com.oneops.cms.md.domain.CmsRelationAttribute;
import com.oneops.cms.md.service.CmsMdProcessor;
import com.oneops.cms.util.CIValidationResult;
import com.oneops.cms.util.CmsDJValidator;
import com.oneops.cms.util.CmsError;
import com.oneops.cms.util.CmsUtil;
import com.oneops.transistor.exceptions.TransistorException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import org.apache.log4j.Logger;

public class BomRfcBulkProcessor {
	static Logger logger = Logger.getLogger(BomRfcBulkProcessor.class);
	
    private static final Map<String, Integer> priorityMap = new HashMap<String, Integer>();
    static {
    	//priorityMap.put("Compute", 2);
    	//priorityMap.put("Storage", 2);
    	priorityMap.put("Keypair", 1);
    }
    
    private static final int priorityMax = 1;
    
    private static final String BOM_CLOUD_RELATION_NAME = "base.DeployedTo";
    private static final String BOM_REALIZED_RELATION_NAME = "base.RealizedAs";
    private static final String BOM_DEPENDS_ON_RELATION_NAME = "bom.DependsOn";
    private static final String BOM_MANAGED_VIA_RELATION_NAME = "bom.ManagedVia";
    private static final boolean ENABLE_BFS_OPTIMIZATION = Boolean.valueOf(System.getProperty("com.oneops.transistor.bfsOptimization", "true"));
    private static final int MAX_RECUSION_DEPTH = Integer.valueOf(System.getProperty("com.oneops.transistor.MaxRecursion", "150"));
    private static final int MAX_NUM_OF_EDGES = Integer.valueOf(System.getProperty("com.oneops.transistor.MaxEdges", "100000"));
    private static final String CONVERGE_RELATION_ATTRIBUTE = "converge";
    private static final String DISABLE_BFS_VAR_NAME= "DISABLE_BFS";
    
	private CmsCmProcessor cmProcessor;
	private CmsMdProcessor mdProcessor;
	private CmsRfcProcessor rfcProcessor;
	private CmsCmRfcMrgProcessor cmRfcMrgProcessor;
	private CmsDJValidator djValidator;
	private Gson gson = new Gson();
	//private CmsNsManager nsManager;
	private TransUtil trUtil;
	private CmsUtil cmsUtil;
	
	public void setCmsUtil(CmsUtil cmsUtil) {
		this.cmsUtil = cmsUtil;
	}
	public void setTrUtil(TransUtil trUtil) {
		this.trUtil = trUtil;
	}
	
	public void setCmProcessor(CmsCmProcessor cmProcessor) {
		this.cmProcessor = cmProcessor;
	}

	public void setMdProcessor(CmsMdProcessor mdProcessor) {
		this.mdProcessor = mdProcessor;
	}

	public void setRfcProcessor(CmsRfcProcessor rfcProcessor) {
		this.rfcProcessor = rfcProcessor;
	}

	public void setCmRfcMrgProcessor(CmsCmRfcMrgProcessor cmRfcMrgProcessor) {
		this.cmRfcMrgProcessor = cmRfcMrgProcessor;
	}
	
	public void setDjValidator(CmsDJValidator djValidator) {
		this.djValidator = djValidator;
	}
	
	public int processManifestPlatform(CmsCI platformCi, CmsCIRelation bindingRel, String nsPath, int startExecOrder, Map<String,String> globalVars, Map<String,String> cloudVars,String userId, boolean usePercent){
		return processManifestPlatform(platformCi, bindingRel, nsPath, startExecOrder, globalVars, cloudVars, userId, false, usePercent);
	}
	
	public int processManifestPlatform(CmsCI platformCi, CmsCIRelation bindingRel, String nsPath, int startExecOrder, Map<String,String> globalVars, Map<String,String> cloudVars, String userId, boolean createPlatNs, boolean usePercent){
		
		if (startExecOrder <= priorityMax) startExecOrder = priorityMax+1;
		
		long startingTime = System.currentTimeMillis();
		int maxExecOrder = 0;
		if (createPlatNs) {
			if (platformCi.getCiClassName().equals("manifest.Iaas")) {
				nsPath = nsPath + "/" + platformCi.getCiName();
			} else {
				nsPath = nsPath + "/" + platformCi.getCiName() + "/" + platformCi.getAttribute("major_version").getDjValue();
			}
			trUtil.verifyAndCreateNS(nsPath);
		}

		logger.info(nsPath + " >>> Start working on " + platformCi.getCiName() + ", cloud - " + bindingRel.getToCi().getCiName());
		Map<String,String> localVars = cmsUtil.getLocalVars(platformCi);
		
		List<CmsCIRelation> mfstPlatComponents = cmProcessor.getFromCIRelations(platformCi.getCiId(), null, "Requires", null);
		if (mfstPlatComponents.size() > 0) {
			Long releaseId  = rfcProcessor.getOpenReleaseIdByNs(getReleaseNs(nsPath).toString(), null, userId);

			String manifestNs = mfstPlatComponents.get(0).getNsPath();
			boolean isPartial = isPartialDeployment(manifestNs);
			
			List<BomRfc> boms = new ArrayList<BomRfc>();
			Map<String, List<BomRfc>> mfstId2nodeId = new HashMap<String,List<BomRfc>>();
			
			CmsCI startingPoint = mfstPlatComponents.get(0).getToCi(); 
			Map<Long,Map<String,List<CmsCIRelation>>> manifestDependsOnRels = new HashMap<Long,Map<String,List<CmsCIRelation>>>();

			List<CmsCIRelation> dependsOn = cmProcessor.getCIRelations(startingPoint.getNsPath(), null, "DependsOn", null, null);
			Map<Long, List<CmsCIRelation>> depOnFromMap = new HashMap<>();
			Map<Long, List<CmsCIRelation>> depOnToMap = new HashMap<>(); 
			for (CmsCIRelation doRel: dependsOn){
				depOnFromMap.computeIfAbsent(doRel.getFromCiId(), k -> new ArrayList<>());
				depOnFromMap.get(doRel.getFromCiId()).add(doRel);
				depOnToMap.computeIfAbsent(doRel.getToCiId(), k -> new ArrayList<>());
				depOnToMap.get(doRel.getToCiId()).add(doRel);
			}
			
			while (startingPoint != null) {
				BomRfc newBom = bootstrapNewBom(startingPoint, bindingRel.getToCiId(), 1);
				boms.add(newBom);	
				mfstId2nodeId.put(String.valueOf(newBom.manifestCiId) + "-" + 1, new ArrayList<BomRfc>(Collections.singletonList(newBom)));
				long startTime = System.currentTimeMillis();
				boms.addAll(processNode(newBom, bindingRel, mfstId2nodeId, manifestDependsOnRels, 1, usePercent, 1, depOnFromMap, depOnToMap));
				logger.info("Time to process Nodes:"+ (System.currentTimeMillis()-startTime));
				startingPoint = getStartingPoint(mfstPlatComponents, boms);
			}
			// this is needed to work around ibatis 
			// if there is no any updates within current transaction 
			// ibatis would not return a new object as query result but instead a ref to the previousely created one
			// if it was modified outside - the changes will not be reset
			for(BomRfc bom : boms) {
				bom.mfstCi = trUtil.cloneCI(bom.mfstCi);
			}	
			//process vars
			processVars(boms, cloudVars, globalVars, localVars);
			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", starting creating rfcs");

			
			long bomCreationStartTime = System.currentTimeMillis();
			ExistingRels existingRels = new ExistingRels(nsPath);
			Map<String, CmsCI> existingCIs = getExistingCis(bindingRel.getToCiId(), nsPath);
			logger.info("existing load:"+(System.currentTimeMillis()-bomCreationStartTime));
			Map<String, CmsRfcCI> existingRFCs = getOpenRFCs(nsPath);
			logger.info("open load:"+(System.currentTimeMillis()-bomCreationStartTime));
			maxExecOrder = createBomRfcsAndRels(boms, nsPath, bindingRel, startExecOrder, isPartial, userId, existingRels, existingCIs, existingRFCs, releaseId);

			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", Done with main RFCs and relations, time spent - " + (System.currentTimeMillis() - bomCreationStartTime));

			Map<Long, List<BomRfc>> bomsMap = buildMfstToBomRfcMap(boms);
			
			long mngviaStartTime = System.currentTimeMillis();
			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", processing managed via");
			processManagedViaRels(mfstPlatComponents,bomsMap,nsPath, userId, existingRels, releaseId, depOnFromMap);
			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", Done with managed via, time spent - " + (System.currentTimeMillis() - mngviaStartTime));

			
			long secByStartTime = System.currentTimeMillis();
			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", processing secured by");
			processSecuredByRels(mfstPlatComponents,bomsMap,nsPath, userId, existingRels, releaseId);
			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", Done with secured by, time spent - " + (System.currentTimeMillis() - secByStartTime));
			
			long entryPointStartTime = System.currentTimeMillis();
			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", processing entry point");
			processEntryPointRel(platformCi.getCiId(),bomsMap, nsPath, userId, existingRels);
			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", Done with entry point, time spent - " + (System.currentTimeMillis() - entryPointStartTime));
			
			if (!usePercent || !isPartial) {
				if (maxExecOrder == 0) maxExecOrder++;
				long obsoleteStartTime = System.currentTimeMillis();
				logger.info(nsPath + " >>> " + platformCi.getCiName() + ", finding obsolete boms");
				maxExecOrder = findObsolete(boms, bindingRel, nsPath, maxExecOrder, existingCIs, userId, false);
				logger.info(nsPath + " >>> " + platformCi.getCiName() + ", Done with obsolete boms, time spent - " + (System.currentTimeMillis() - obsoleteStartTime));
			}
			if (logger.isDebugEnabled()) {
				for(BomRfc bom : boms) {
					logger.debug(bom.ciName + "::" + bom.execOrder);
				}
			}
			if (rfcProcessor.getRfcCount(releaseId) == 0) {  // clean up redundant release
				logger.info("No release because rfc count is 0. Cleaning up release.");
				rfcProcessor.deleteRelease(releaseId);
			}
			//help gc a little bit
			existingRels = null;
			existingCIs = null;
			existingRFCs = null;

		}
		
		long timeTook = System.currentTimeMillis() - startingTime;
		logger.info(nsPath + ">>> Done with " + platformCi.getCiName() + ", cloud - " + bindingRel.getToCi().getCiName() + ", Time to process - " + timeTook + " ms.");
		return maxExecOrder;
	}
	
	private boolean isPartialDeployment(String manifestNs) {
		List<CmsCIRelation> dependsOns = cmProcessor.getCIRelationsNaked(manifestNs, "manifest.DependsOn", null, null, null);
		
		for (CmsCIRelation rel : dependsOns) {
			if (rel.getAttribute("pct_dpmt") != null && !"100".equals(rel.getAttribute("pct_dpmt").getDjValue())){
				return true;
			}
		}
		return false;
	}
	
	private CmsCI getStartingPoint(List<CmsCIRelation> mfstPlatComponents, List<BomRfc> boms) {
		Set<Long> processedNodes = new HashSet<Long>();
		for (BomRfc bom : boms) {
			processedNodes.add(bom.manifestCiId);
		}
		for (CmsCIRelation manifestRel : mfstPlatComponents) {
			if (!processedNodes.contains(manifestRel.getToCiId())) {
				return manifestRel.getToCi();
			}
		}
		return null;
	}

	private void processVars(List<BomRfc> boms, Map<String,String> cloudVars, Map<String,String> globalVars, Map<String,String> localVars) {
        ExceptionConsolidator ec = CIValidationException.consolidator(CmsError.TRANSISTOR_CM_ATTRIBUTE_HAS_BAD_GLOBAL_VAR_REF,cmsUtil.getCountOfErrorsToReport());
		for (BomRfc bom : boms) {
			ec.invokeChecked(()-> trUtil.processAllVars(bom.mfstCi, cloudVars, globalVars, localVars));
		}
		ec.rethrowExceptionIfNeeded();
	}
	
	private int findObsolete(List<BomRfc> newBoms, CmsCIRelation bindingRel, String nsPath, int startingExecOrder, Map<String, CmsCI> existingCIs,String userId,  boolean global) {
		logger.info(nsPath + " >>> finding cis to delete..." );
		long startTime = System.currentTimeMillis();
		int maxExecOrder = startingExecOrder;
		Map<String, BomRfc> bomMap = new HashMap<String, BomRfc>();
		for (BomRfc bom : newBoms) {
			bomMap.put(bom.ciName, bom);
		}

		List<CmsCI> existingCis = new ArrayList<CmsCI>(existingCIs.values());
		
		Map<Long, CmsCI> obsoleteCisMap = new HashMap<Long, CmsCI>();
		for (CmsCI ci : existingCis) {
			if (!bomMap.containsKey(ci.getCiName())) {
				logger.info("This ci should be deleted - " + ci.getCiName());
				obsoleteCisMap.put(ci.getCiId(), ci);<commit_msg>add variable validation for attachments<commit_after>/*******************************************************************************
 *  
 *   Copyright 2015 Walmart, Inc.
 *  
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *  
 *       http://www.apache.org/licenses/LICENSE-2.0
 *  
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 *  
 *******************************************************************************/
package com.oneops.transistor.service;

import com.google.gson.Gson;
import com.oneops.cms.cm.domain.CmsCI;
import com.oneops.cms.cm.domain.CmsCIAttribute;
import com.oneops.cms.cm.domain.CmsCIRelation;
import com.oneops.cms.cm.domain.CmsCIRelationAttribute;
import com.oneops.cms.cm.domain.CmsLink;
import com.oneops.cms.cm.service.CmsCmProcessor;
import com.oneops.cms.dj.domain.CmsRfcAttribute;
import com.oneops.cms.dj.domain.CmsRfcCI;
import com.oneops.cms.dj.domain.CmsRfcRelation;
import com.oneops.cms.dj.domain.RfcHint;
import com.oneops.cms.dj.service.CmsCmRfcMrgProcessor;
import com.oneops.cms.dj.service.CmsRfcProcessor;
import com.oneops.cms.exceptions.CIValidationException;
import com.oneops.cms.exceptions.DJException;
import com.oneops.cms.exceptions.ExceptionConsolidator;
import com.oneops.cms.md.domain.CmsClazz;
import com.oneops.cms.md.domain.CmsClazzAttribute;
import com.oneops.cms.md.domain.CmsRelation;
import com.oneops.cms.md.domain.CmsRelationAttribute;
import com.oneops.cms.md.service.CmsMdProcessor;
import com.oneops.cms.util.CIValidationResult;
import com.oneops.cms.util.CmsDJValidator;
import com.oneops.cms.util.CmsError;
import com.oneops.cms.util.CmsUtil;
import com.oneops.transistor.exceptions.TransistorException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import org.apache.log4j.Logger;

public class BomRfcBulkProcessor {
	static Logger logger = Logger.getLogger(BomRfcBulkProcessor.class);
	
    private static final Map<String, Integer> priorityMap = new HashMap<String, Integer>();
    static {
    	//priorityMap.put("Compute", 2);
    	//priorityMap.put("Storage", 2);
    	priorityMap.put("Keypair", 1);
    }
    
    private static final int priorityMax = 1;
    
    private static final String BOM_CLOUD_RELATION_NAME = "base.DeployedTo";
    private static final String BOM_REALIZED_RELATION_NAME = "base.RealizedAs";
    private static final String BOM_DEPENDS_ON_RELATION_NAME = "bom.DependsOn";
    private static final String BOM_MANAGED_VIA_RELATION_NAME = "bom.ManagedVia";
    private static final boolean ENABLE_BFS_OPTIMIZATION = Boolean.valueOf(System.getProperty("com.oneops.transistor.bfsOptimization", "true"));
    private static final int MAX_RECUSION_DEPTH = Integer.valueOf(System.getProperty("com.oneops.transistor.MaxRecursion", "150"));
    private static final int MAX_NUM_OF_EDGES = Integer.valueOf(System.getProperty("com.oneops.transistor.MaxEdges", "100000"));
    private static final String CONVERGE_RELATION_ATTRIBUTE = "converge";
    private static final String DISABLE_BFS_VAR_NAME= "DISABLE_BFS";
    
	private CmsCmProcessor cmProcessor;
	private CmsMdProcessor mdProcessor;
	private CmsRfcProcessor rfcProcessor;
	private CmsCmRfcMrgProcessor cmRfcMrgProcessor;
	private CmsDJValidator djValidator;
	private Gson gson = new Gson();
	//private CmsNsManager nsManager;
	private TransUtil trUtil;
	private CmsUtil cmsUtil;
	
	public void setCmsUtil(CmsUtil cmsUtil) {
		this.cmsUtil = cmsUtil;
	}
	public void setTrUtil(TransUtil trUtil) {
		this.trUtil = trUtil;
	}
	
	public void setCmProcessor(CmsCmProcessor cmProcessor) {
		this.cmProcessor = cmProcessor;
	}

	public void setMdProcessor(CmsMdProcessor mdProcessor) {
		this.mdProcessor = mdProcessor;
	}

	public void setRfcProcessor(CmsRfcProcessor rfcProcessor) {
		this.rfcProcessor = rfcProcessor;
	}

	public void setCmRfcMrgProcessor(CmsCmRfcMrgProcessor cmRfcMrgProcessor) {
		this.cmRfcMrgProcessor = cmRfcMrgProcessor;
	}
	
	public void setDjValidator(CmsDJValidator djValidator) {
		this.djValidator = djValidator;
	}
	
	public int processManifestPlatform(CmsCI platformCi, CmsCIRelation bindingRel, String nsPath, int startExecOrder, Map<String,String> globalVars, Map<String,String> cloudVars,String userId, boolean usePercent){
		return processManifestPlatform(platformCi, bindingRel, nsPath, startExecOrder, globalVars, cloudVars, userId, false, usePercent);
	}
	
	public int processManifestPlatform(CmsCI platformCi, CmsCIRelation bindingRel, String nsPath, int startExecOrder, Map<String,String> globalVars, Map<String,String> cloudVars, String userId, boolean createPlatNs, boolean usePercent){
		
		if (startExecOrder <= priorityMax) startExecOrder = priorityMax+1;
		
		long startingTime = System.currentTimeMillis();
		int maxExecOrder = 0;
		if (createPlatNs) {
			if (platformCi.getCiClassName().equals("manifest.Iaas")) {
				nsPath = nsPath + "/" + platformCi.getCiName();
			} else {
				nsPath = nsPath + "/" + platformCi.getCiName() + "/" + platformCi.getAttribute("major_version").getDjValue();
			}
			trUtil.verifyAndCreateNS(nsPath);
		}

		logger.info(nsPath + " >>> Start working on " + platformCi.getCiName() + ", cloud - " + bindingRel.getToCi().getCiName());
		Map<String,String> localVars = cmsUtil.getLocalVars(platformCi);
		
		List<CmsCIRelation> mfstPlatComponents = cmProcessor.getFromCIRelations(platformCi.getCiId(), null, "Requires", null);
		if (mfstPlatComponents.size() > 0) {
			Long releaseId  = rfcProcessor.getOpenReleaseIdByNs(getReleaseNs(nsPath).toString(), null, userId);

			String manifestNs = mfstPlatComponents.get(0).getNsPath();
			boolean isPartial = isPartialDeployment(manifestNs);
			
			List<BomRfc> boms = new ArrayList<BomRfc>();
			Map<String, List<BomRfc>> mfstId2nodeId = new HashMap<String,List<BomRfc>>();
			
			CmsCI startingPoint = mfstPlatComponents.get(0).getToCi(); 
			Map<Long,Map<String,List<CmsCIRelation>>> manifestDependsOnRels = new HashMap<Long,Map<String,List<CmsCIRelation>>>();

			List<CmsCIRelation> dependsOn = cmProcessor.getCIRelations(startingPoint.getNsPath(), null, "DependsOn", null, null);
			Map<Long, List<CmsCIRelation>> depOnFromMap = new HashMap<>();
			Map<Long, List<CmsCIRelation>> depOnToMap = new HashMap<>(); 
			for (CmsCIRelation doRel: dependsOn){
				depOnFromMap.computeIfAbsent(doRel.getFromCiId(), k -> new ArrayList<>());
				depOnFromMap.get(doRel.getFromCiId()).add(doRel);
				depOnToMap.computeIfAbsent(doRel.getToCiId(), k -> new ArrayList<>());
				depOnToMap.get(doRel.getToCiId()).add(doRel);
			}
			
			while (startingPoint != null) {
				BomRfc newBom = bootstrapNewBom(startingPoint, bindingRel.getToCiId(), 1);
				boms.add(newBom);	
				mfstId2nodeId.put(String.valueOf(newBom.manifestCiId) + "-" + 1, new ArrayList<BomRfc>(Collections.singletonList(newBom)));
				long startTime = System.currentTimeMillis();
				boms.addAll(processNode(newBom, bindingRel, mfstId2nodeId, manifestDependsOnRels, 1, usePercent, 1, depOnFromMap, depOnToMap));
				logger.info("Time to process Nodes:"+ (System.currentTimeMillis()-startTime));
				startingPoint = getStartingPoint(mfstPlatComponents, boms);
			}
			// this is needed to work around ibatis 
			// if there is no any updates within current transaction 
			// ibatis would not return a new object as query result but instead a ref to the previousely created one
			// if it was modified outside - the changes will not be reset
			for(BomRfc bom : boms) {
				bom.mfstCi = trUtil.cloneCI(bom.mfstCi);
			}	

			List<CmsCI> attachments = cmProcessor.getCIRelations(platformCi.getNsPath(),"manifest.EscortedBy", null, null, null).stream().map(CmsCIRelation::getToCi).collect(Collectors.toList());
			// get attachments to verify attachments variables too`
			processVars(boms, attachments, cloudVars, globalVars, localVars);

			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", starting creating rfcs");

			
			long bomCreationStartTime = System.currentTimeMillis();
			ExistingRels existingRels = new ExistingRels(nsPath);
			Map<String, CmsCI> existingCIs = getExistingCis(bindingRel.getToCiId(), nsPath);
			logger.info("existing load:"+(System.currentTimeMillis()-bomCreationStartTime));
			Map<String, CmsRfcCI> existingRFCs = getOpenRFCs(nsPath);
			logger.info("open load:"+(System.currentTimeMillis()-bomCreationStartTime));
			maxExecOrder = createBomRfcsAndRels(boms, nsPath, bindingRel, startExecOrder, isPartial, userId, existingRels, existingCIs, existingRFCs, releaseId);

			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", Done with main RFCs and relations, time spent - " + (System.currentTimeMillis() - bomCreationStartTime));

			Map<Long, List<BomRfc>> bomsMap = buildMfstToBomRfcMap(boms);
			
			long mngviaStartTime = System.currentTimeMillis();
			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", processing managed via");
			processManagedViaRels(mfstPlatComponents,bomsMap,nsPath, userId, existingRels, releaseId, depOnFromMap);
			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", Done with managed via, time spent - " + (System.currentTimeMillis() - mngviaStartTime));

			
			long secByStartTime = System.currentTimeMillis();
			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", processing secured by");
			processSecuredByRels(mfstPlatComponents,bomsMap,nsPath, userId, existingRels, releaseId);
			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", Done with secured by, time spent - " + (System.currentTimeMillis() - secByStartTime));
			
			long entryPointStartTime = System.currentTimeMillis();
			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", processing entry point");
			processEntryPointRel(platformCi.getCiId(),bomsMap, nsPath, userId, existingRels);
			logger.info(nsPath + " >>> " + platformCi.getCiName() + ", Done with entry point, time spent - " + (System.currentTimeMillis() - entryPointStartTime));
			
			if (!usePercent || !isPartial) {
				if (maxExecOrder == 0) maxExecOrder++;
				long obsoleteStartTime = System.currentTimeMillis();
				logger.info(nsPath + " >>> " + platformCi.getCiName() + ", finding obsolete boms");
				maxExecOrder = findObsolete(boms, bindingRel, nsPath, maxExecOrder, existingCIs, userId, false);
				logger.info(nsPath + " >>> " + platformCi.getCiName() + ", Done with obsolete boms, time spent - " + (System.currentTimeMillis() - obsoleteStartTime));
			}
			if (logger.isDebugEnabled()) {
				for(BomRfc bom : boms) {
					logger.debug(bom.ciName + "::" + bom.execOrder);
				}
			}
			if (rfcProcessor.getRfcCount(releaseId) == 0) {  // clean up redundant release
				logger.info("No release because rfc count is 0. Cleaning up release.");
				rfcProcessor.deleteRelease(releaseId);
			}
			//help gc a little bit
			existingRels = null;
			existingCIs = null;
			existingRFCs = null;

		}
		
		long timeTook = System.currentTimeMillis() - startingTime;
		logger.info(nsPath + ">>> Done with " + platformCi.getCiName() + ", cloud - " + bindingRel.getToCi().getCiName() + ", Time to process - " + timeTook + " ms.");
		return maxExecOrder;
	}
	
	private boolean isPartialDeployment(String manifestNs) {
		List<CmsCIRelation> dependsOns = cmProcessor.getCIRelationsNaked(manifestNs, "manifest.DependsOn", null, null, null);
		
		for (CmsCIRelation rel : dependsOns) {
			if (rel.getAttribute("pct_dpmt") != null && !"100".equals(rel.getAttribute("pct_dpmt").getDjValue())){
				return true;
			}
		}
		return false;
	}
	
	private CmsCI getStartingPoint(List<CmsCIRelation> mfstPlatComponents, List<BomRfc> boms) {
		Set<Long> processedNodes = new HashSet<Long>();
		for (BomRfc bom : boms) {
			processedNodes.add(bom.manifestCiId);
		}
		for (CmsCIRelation manifestRel : mfstPlatComponents) {
			if (!processedNodes.contains(manifestRel.getToCiId())) {
				return manifestRel.getToCi();
			}
		}
		return null;
	}

	private void processVars(List<BomRfc> boms, List<CmsCI> attachments, Map<String,String> cloudVars, Map<String,String> globalVars, Map<String,String> localVars) {
        ExceptionConsolidator ec = CIValidationException.consolidator(CmsError.TRANSISTOR_CM_ATTRIBUTE_HAS_BAD_GLOBAL_VAR_REF,cmsUtil.getCountOfErrorsToReport());
		for (BomRfc bom : boms) {
			ec.invokeChecked(()-> trUtil.processAllVars(bom.mfstCi, cloudVars, globalVars, localVars));
		}
		for (CmsCI att: attachments){
			ec.invokeChecked(()-> trUtil.processAllVars(att, cloudVars, globalVars, localVars));
		}
		ec.rethrowExceptionIfNeeded();
	}
	
	private int findObsolete(List<BomRfc> newBoms, CmsCIRelation bindingRel, String nsPath, int startingExecOrder, Map<String, CmsCI> existingCIs,String userId,  boolean global) {
		logger.info(nsPath + " >>> finding cis to delete..." );
		long startTime = System.currentTimeMillis();
		int maxExecOrder = startingExecOrder;
		Map<String, BomRfc> bomMap = new HashMap<String, BomRfc>();
		for (BomRfc bom : newBoms) {
			bomMap.put(bom.ciName, bom);
		}

		List<CmsCI> existingCis = new ArrayList<CmsCI>(existingCIs.values());
		
		Map<Long, CmsCI> obsoleteCisMap = new HashMap<Long, CmsCI>();
		for (CmsCI ci : existingCis) {
			if (!bomMap.containsKey(ci.getCiName())) {
				logger.info("This ci should be deleted - " + ci.getCiName());
				obsoleteCisMap.put(ci.getCiId(), ci);
{'fraction_non_alphanumeric': 0.13045225396361512, 'fraction_numerical': 0.002922481186527362, 'mean_word_length': 5.044380657981894, 'pattern_counts': {'":': 2, '<': 109, '<?xml version=': 0, '>': 194, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3023882', 'n_tokens_mistral': 9841, 'n_tokens_neox': 9002, 'n_words': 1820}