<commit_before>package com.capitalone.dashboard.collector;


import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.bson.types.ObjectId;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;

import com.capitalone.dashboard.model.ArtifactoryCollector;
import com.capitalone.dashboard.model.ArtifactoryRepo;
import com.capitalone.dashboard.model.BinaryArtifact;
import com.capitalone.dashboard.repository.ArtifactoryCollectorRepository;
import com.capitalone.dashboard.repository.ArtifactoryRepoRepository;
import com.capitalone.dashboard.repository.BaseCollectorRepository;
import com.capitalone.dashboard.repository.BinaryArtifactRepository;
import com.google.common.collect.Iterables;

@Component
public class ArtifactoryCollectorTask extends CollectorTask<ArtifactoryCollector>{
	private final ArtifactoryCollectorRepository artifactoryCollectorRepository;
	private final ArtifactoryRepoRepository artifactoryRepoRepository;
	private final ArtifactoryClient artifactoryClient;
	private final ArtifactorySettings artifactorySettings;
	private final BinaryArtifactRepository binaryArtifactRepository;
	
	@Autowired
	public ArtifactoryCollectorTask(TaskScheduler taskScheduler,
									ArtifactoryCollectorRepository artifactoryCollectorRepository,
									ArtifactoryRepoRepository artifactoryRepoRepository,
									BinaryArtifactRepository binaryArtifactRepository,
									ArtifactoryClient artifactoryClient,
									ArtifactorySettings artifactorySettings) {
		super(taskScheduler, "Artifactory");
		this.artifactoryCollectorRepository = artifactoryCollectorRepository;
		this.artifactoryRepoRepository = artifactoryRepoRepository;
		this.binaryArtifactRepository = binaryArtifactRepository;
		this.artifactoryClient = artifactoryClient;
		this.artifactorySettings = artifactorySettings;
	}
	
    @Override
    public ArtifactoryCollector getCollector() {
        return ArtifactoryCollector.prototype(artifactorySettings.getServers());
    }
    
    @Override
    public BaseCollectorRepository<ArtifactoryCollector> getCollectorRepository() {
        return artifactoryCollectorRepository;
    }

    @Override
    public String getCron() {
        return artifactorySettings.getCron();
    }
    
    @Override
    public void collect(ArtifactoryCollector collector) {
    	Set<ObjectId> udId = new HashSet<>();
        udId.add(collector.getId());
        List<ArtifactoryRepo> existingRepos = artifactoryRepoRepository.findByCollectorIdIn(udId);
        List<ArtifactoryRepo> activeRepos = new ArrayList<>();
        List<String> activeServers = new ArrayList<>();
        activeServers.addAll(collector.getArtifactoryServers());
        
        clean(collector, existingRepos);
        
        List<String> instanceUrls = collector.getArtifactoryServers();
        for (int i = 0; i < instanceUrls.size(); i++) { 
    		long start = System.currentTimeMillis();
    		
    		logBanner(instanceUrls.get(i));
    		
    		List<ArtifactoryRepo> repos = artifactoryClient.getRepos(instanceUrls.get(i));
    		log("Fetched repos", start);
    		activeRepos.addAll(repos);
    		addNewRepos(repos, existingRepos, collector);
    		addNewArtifacts(enabledRepos(collector, instanceUrls.get(i)));
    		
    		log("Finished", start);
    	}
    	
    	// Delete repos that will be no longer collected
    	deleteUnwantedRepos(activeRepos, existingRepos, activeServers, collector);
    }
    
    /**
     * Clean up unused artifactory collector items
     *
     * @param collector the {@link ArtifactoryCollector}
     */
    private void clean(ArtifactoryCollector collector, List<ArtifactoryRepo> existingRepos) {
    	// find the server url's to collect from
    	Set<String> serversToBeCollected = new HashSet<>();
    	serversToBeCollected.addAll(artifactorySettings.getServers());
    	
    	// find the repos to collect from each server url above
    	List<Set<String>> repoNamesToBeCollected = new ArrayList<Set<String>>();
    	List<String[]> allRepos = artifactorySettings.getRepos();
        for (int i = 0; i < allRepos.size(); i++) {
        	Set<String> reposSet = new HashSet<>();
        	if (allRepos.get(i) != null) { 		
        		reposSet.addAll(Arrays.asList(allRepos.get(i)));
        	}
        	repoNamesToBeCollected.add(reposSet);
        }
        
        assert(serversToBeCollected.size() == repoNamesToBeCollected.size());
        
        List<ArtifactoryRepo> stateChangeRepoList = new ArrayList<>();
        for (ArtifactoryRepo repo : existingRepos) {
            if ((repo.isEnabled() && (!collector.getId().equals(repo.getCollectorId()) 
            			|| !serversToBeCollected.contains(repo.getInstanceUrl()) 
            			|| !repoNamesToBeCollected.get(artifactorySettings.getServers().indexOf(repo.getInstanceUrl())).contains(repo.getRepoName()))) ||  // if it was enabled but not to be collected
                    (!repo.isEnabled() && (collector.getId().equals(repo.getCollectorId()) 
                    	&& serversToBeCollected.contains(repo.getInstanceUrl()) 
                    	&& repoNamesToBeCollected.get(artifactorySettings.getServers().indexOf(repo.getInstanceUrl())).contains(repo.getRepoName())))) { // OR it was disabled and now is to be collected
                repo.setEnabled((collector.getId().equals(repo.getCollectorId()) 
                				&& serversToBeCollected.contains(repo.getInstanceUrl()) 
                				&& repoNamesToBeCollected.get(artifactorySettings.getServers().indexOf(repo.getInstanceUrl())).contains(repo.getRepoName())));
                stateChangeRepoList.add(repo);
            }
        }
        if (!CollectionUtils.isEmpty(stateChangeRepoList)) {
        	artifactoryRepoRepository.save(stateChangeRepoList);
        }
    }
    
    private void deleteUnwantedRepos(List<ArtifactoryRepo> activeRepos, List<ArtifactoryRepo> existingRepos, List<String> activeServers, ArtifactoryCollector collector) {
    	List<ArtifactoryRepo> deleteList = new ArrayList<>();
    	
    	for (ArtifactoryRepo repo : existingRepos) {
            // if we have a collector item for the repo in repository but it's server is not what we collect, remove it.
            if (!collector.getArtifactoryServers().contains(repo.getInstanceUrl())) {
            	deleteList.add(repo);
            }

            //if the collector id of the collector item for the repo does not match with the collector ID, delete it.
            if (!repo.getCollectorId().equals(collector.getId())) {
            	deleteList.add(repo);
            }

            // this is to handle repos that have been deleted from servers. Will get 404 if we don't delete them.
            if (activeServers.contains(repo.getInstanceUrl()) && !activeRepos.contains(repo)) {
            	deleteList.add(repo);
            }

        }
        if (!CollectionUtils.isEmpty(deleteList)) {
        	artifactoryRepoRepository.delete(deleteList);
        }
    }
    	<commit_msg>Fixed to not delete unwanted repos, just disable them instead<commit_after>package com.capitalone.dashboard.collector;


import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.bson.types.ObjectId;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;

import com.capitalone.dashboard.model.ArtifactoryCollector;
import com.capitalone.dashboard.model.ArtifactoryRepo;
import com.capitalone.dashboard.model.BinaryArtifact;
import com.capitalone.dashboard.repository.ArtifactoryCollectorRepository;
import com.capitalone.dashboard.repository.ArtifactoryRepoRepository;
import com.capitalone.dashboard.repository.BaseCollectorRepository;
import com.capitalone.dashboard.repository.BinaryArtifactRepository;
import com.google.common.collect.Iterables;

@Component
public class ArtifactoryCollectorTask extends CollectorTask<ArtifactoryCollector>{
	private final ArtifactoryCollectorRepository artifactoryCollectorRepository;
	private final ArtifactoryRepoRepository artifactoryRepoRepository;
	private final ArtifactoryClient artifactoryClient;
	private final ArtifactorySettings artifactorySettings;
	private final BinaryArtifactRepository binaryArtifactRepository;
	
	@Autowired
	public ArtifactoryCollectorTask(TaskScheduler taskScheduler,
									ArtifactoryCollectorRepository artifactoryCollectorRepository,
									ArtifactoryRepoRepository artifactoryRepoRepository,
									BinaryArtifactRepository binaryArtifactRepository,
									ArtifactoryClient artifactoryClient,
									ArtifactorySettings artifactorySettings) {
		super(taskScheduler, "Artifactory");
		this.artifactoryCollectorRepository = artifactoryCollectorRepository;
		this.artifactoryRepoRepository = artifactoryRepoRepository;
		this.binaryArtifactRepository = binaryArtifactRepository;
		this.artifactoryClient = artifactoryClient;
		this.artifactorySettings = artifactorySettings;
	}
	
    @Override
    public ArtifactoryCollector getCollector() {
        return ArtifactoryCollector.prototype(artifactorySettings.getServers());
    }
    
    @Override
    public BaseCollectorRepository<ArtifactoryCollector> getCollectorRepository() {
        return artifactoryCollectorRepository;
    }

    @Override
    public String getCron() {
        return artifactorySettings.getCron();
    }
    
    @Override
    public void collect(ArtifactoryCollector collector) {
    	Set<ObjectId> udId = new HashSet<>();
        udId.add(collector.getId());
        List<ArtifactoryRepo> existingRepos = artifactoryRepoRepository.findByCollectorIdIn(udId);
        List<ArtifactoryRepo> activeRepos = new ArrayList<>();
        List<String> activeServers = new ArrayList<>();
        activeServers.addAll(collector.getArtifactoryServers());
        
        clean(collector, existingRepos);
        
        List<String> instanceUrls = collector.getArtifactoryServers();
        for (int i = 0; i < instanceUrls.size(); i++) { 
    		long start = System.currentTimeMillis();
    		
    		logBanner(instanceUrls.get(i));
    		
    		List<ArtifactoryRepo> repos = artifactoryClient.getRepos(instanceUrls.get(i));
    		log("Fetched repos", start);
    		activeRepos.addAll(repos);
    		addNewRepos(repos, existingRepos, collector);
    		addNewArtifacts(enabledRepos(collector, instanceUrls.get(i)));
    		
    		log("Finished", start);
    	}
    }
    
    /**
     * Clean up unused artifactory collector items
     *
     * @param collector the {@link ArtifactoryCollector}
     */
    private void clean(ArtifactoryCollector collector, List<ArtifactoryRepo> existingRepos) {
    	// find the server url's to collect from
    	Set<String> serversToBeCollected = new HashSet<>();
    	serversToBeCollected.addAll(artifactorySettings.getServers());
    	
    	// find the repos to collect from each server url above
    	List<Set<String>> repoNamesToBeCollected = new ArrayList<Set<String>>();
    	List<String[]> allRepos = artifactorySettings.getRepos();
        for (int i = 0; i < allRepos.size(); i++) {
        	Set<String> reposSet = new HashSet<>();
        	if (allRepos.get(i) != null) { 		
        		reposSet.addAll(Arrays.asList(allRepos.get(i)));
        	}
        	repoNamesToBeCollected.add(reposSet);
        }
        
        assert(serversToBeCollected.size() == repoNamesToBeCollected.size());
        
        List<ArtifactoryRepo> stateChangeRepoList = new ArrayList<>();
        for (ArtifactoryRepo repo : existingRepos) {
            if ((repo.isEnabled() && (!collector.getId().equals(repo.getCollectorId()) 
            			|| !serversToBeCollected.contains(repo.getInstanceUrl()) 
            			|| !repoNamesToBeCollected.get(artifactorySettings.getServers().indexOf(repo.getInstanceUrl())).contains(repo.getRepoName()))) ||  // if it was enabled but not to be collected
                    (!repo.isEnabled() && (collector.getId().equals(repo.getCollectorId()) 
                    	&& serversToBeCollected.contains(repo.getInstanceUrl()) 
                    	&& repoNamesToBeCollected.get(artifactorySettings.getServers().indexOf(repo.getInstanceUrl())).contains(repo.getRepoName())))) { // OR it was disabled and now is to be collected
                repo.setEnabled((collector.getId().equals(repo.getCollectorId()) 
                				&& serversToBeCollected.contains(repo.getInstanceUrl()) 
                				&& repoNamesToBeCollected.get(artifactorySettings.getServers().indexOf(repo.getInstanceUrl())).contains(repo.getRepoName())));
                stateChangeRepoList.add(repo);
            }
        }
        if (!CollectionUtils.isEmpty(stateChangeRepoList)) {
        	artifactoryRepoRepository.save(stateChangeRepoList);
        }
    }
    	
{'fraction_non_alphanumeric': 0.09633483816665392, 'fraction_numerical': 0.0005356186395286556, 'mean_word_length': 3.6661906461977867, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3878916', 'n_tokens_mistral': 3805, 'n_tokens_neox': 3680, 'n_words': 646}