<commit_before>package mind;

import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.apache.commons.configuration.ConfigurationException;
import org.apache.log4j.Logger;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.InvalidRemoteException;
import org.eclipse.jgit.api.errors.NoHeadException;
import org.eclipse.jgit.api.errors.TransportException;

import com.taskadapter.redmineapi.RedmineException;

public class Analyzer {
	private SonarReader sonarReader;
	private IssueTrackerReader issueTrackerReader;
	private SCMReader scmReader;
	private VersionDAO versionDao;
	private SonarRunnerApi sonarRunner;
	private static org.apache.log4j.Logger log = Logger
			.getLogger(Analyzer.class);

	public Analyzer(SonarReader sonarReader,
			IssueTrackerReader issueTrackerReader, SCMReader scmReader,
			SonarRunnerApi sonarRunner) throws IOException,
			ConfiguredVersionNotExistInSonarException,
			UnequalNumberOfVersionsException {
		this.sonarReader = sonarReader;
		this.issueTrackerReader = issueTrackerReader;
		this.scmReader = scmReader;
		versionDao = new VersionDAO(scmReader, issueTrackerReader, sonarReader);
		this.sonarRunner = sonarRunner;
	}

	public LinkedHashMap<String, HashMap<String, HashMap<String, Integer>>> getTechnicalDebtTable()
			throws ConfigurationException, IOException, InvalidRemoteException,
			TransportException, GitAPIException, RedmineException, VersionIdentifierConflictException, ConfiguredVersionNotExistInSonarException, UnequalNumberOfVersionsException, KeyNotFoundException {
		
		LinkedHashMap<String, HashMap<String, HashMap<String, Integer>>> table = new LinkedHashMap<String, HashMap<String, HashMap<String, Integer>>>();
		
		String previousVersionKey = "0";
		HashMap<String, HashMap<String, Integer>> sizeOfResourcePerVersion = new HashMap<String, HashMap<String,Integer>>();
		HashMap<String, HashMap<String, HashMap<String, Integer>>> violationsOfResourcePerVersion = new HashMap<String, HashMap<String,HashMap<String,Integer>>>();
		
		
		for(String currentVersionKey : versionDao.getKeySet())
		{
			sonarRunner.runSonar(versionDao.getScmVersion(currentVersionKey));
			List<String> resources = sonarReader.getListOfAllResources();
			
			HashMap<String,HashMap<String, Integer>> mapOfNumberOfDefectsRelatedToClassPerVersion = getMapOfNumberOfDefectsRelatedToResource(resources, scmReader.getHeadBranch());
			HashMap<String, HashMap<String, Integer>> resourceRows = new HashMap<String, HashMap<String,Integer>>();
			HashMap<String, Integer> sizeOfResources = new HashMap<String, Integer>();
			HashMap<String, HashMap<String, Integer>> violationsOfResource = new HashMap<String, HashMap<String, Integer>>();
			
			
			for (String resource : resources)
			{
				int numberOfDefectsForThisResourceInThisVersion = mapOfNumberOfDefectsRelatedToClassPerVersion.get(versionDao.getMainKeyVersion(currentVersionKey)).get(resource);
				HashMap<String, Integer> technicalDebtRow = new HashMap<String, Integer>();
				
				Integer numberOfLOCTouched;
				int sizeOfClass = sonarReader.getSizeOfClass(resource);
				Iterator<Map.Entry<String, Integer>> it;
				if(previousVersionKey.equals("0") || !violationsOfResourcePerVersion.get(previousVersionKey).containsKey(resource))
				{
					numberOfLOCTouched = null;
					technicalDebtRow.put("size", 0);
					violationsOfResource.put(resource, sonarReader.getNumberOfViolationsPerRuleEverythingZero());
					it = violationsOfResource.get(resource).entrySet().iterator();
				}else
				{
					numberOfLOCTouched = scmReader.getNumberOfLOCtouched(versionDao.getScmVersion(currentVersionKey), versionDao.getScmVersion(previousVersionKey), resource);
					technicalDebtRow.put("size", sizeOfResourcePerVersion.get(previousVersionKey).get(resource));
					it = violationsOfResourcePerVersion.get(previousVersionKey).get(resource).entrySet().iterator();

				}
		 		while (it.hasNext()) {
		 			Map.Entry<String, Integer> pairs = (Map.Entry<String, Integer>) it.next();
				 			technicalDebtRow.put(pairs.getKey(), pairs.getValue());
				 			it.remove();
				 }
				
				technicalDebtRow.put("numberDefects", numberOfDefectsForThisResourceInThisVersion);
				technicalDebtRow.put("locTouched", numberOfLOCTouched);

				resourceRows.put(resource, technicalDebtRow);
				sizeOfResources.put(resource, sizeOfClass);
				violationsOfResource.put(resource, sonarReader.getNumberOfViolationsPerRule(resource));
				
				sizeOfResourcePerVersion.put(currentVersionKey, sizeOfResources);
				violationsOfResourcePerVersion.put(currentVersionKey, violationsOfResource);
			}
			table.put(currentVersionKey, resourceRows);
			previousVersionKey = currentVersionKey;
		}
		return table;
	}

	// FIXME: TO COMPLICATED AND VERY VERY SLOW
	public HashMap<String, HashMap<String, Integer>> getMapOfNumberOfDefectsRelatedToResource(
			List<String> resources, String branch) throws NoHeadException,
			IOException, GitAPIException, RedmineException,
			VersionIdentifierConflictException, KeyNotFoundException {
		HashMap<String, HashMap<String, Set<Integer>>> mapOfDefectsRelatedToResource = getMapOfDefectsRelatedToResource(
				resources, branch);
		HashMap<String, HashMap<String, Integer>> mapOfNumberOfDefectsRelatedToResource = new HashMap<String, HashMap<String, Integer>>();

		for (Entry<String, HashMap<String, Set<Integer>>> map : mapOfDefectsRelatedToResource
				.entrySet()) {
			HashMap<String, Integer> resourceToNumberOfDefects = new HashMap<String, Integer>();
			for (Entry<String, Set<Integer>> resourceCount : map.getValue()
					.entrySet()) {
				resourceToNumberOfDefects.put(resourceCount.getKey(),
						resourceCount.getValue().size());
				mapOfNumberOfDefectsRelatedToResource.put(map.getKey(),
						resourceToNumberOfDefects);
			}

		}
		return mapOfNumberOfDefectsRelatedToResource;
	}

	private HashMap<String, HashMap<String, Set<Integer>>> getMapOfDefectsRelatedToResource(
			List<String> resources, String branch) throws NoHeadException,
			IOException, GitAPIException, RedmineException,
			VersionIdentifierConflictException, KeyNotFoundException {
		HashMap<String, HashMap<String, Set<Integer>>> mapOfDefectsRelatedToResource = new HashMap<String, HashMap<String, Set<Integer>>>();

		for (String version : versionDao.getKeySet()) {
			HashMap<String, Set<Integer>> resourceToDefectsMap = new HashMap<String, Set<Integer>>();
			for (String resource : resources) {
				Set<Integer> setOfDefectIds = new HashSet<Integer>();
				resourceToDefectsMap.put(resource, setOfDefectIds);
			}
			mapOfDefectsRelatedToResource.put(
					versionDao.getMainKeyVersion(version),
					new HashMap<String, Set<Integer>>(resourceToDefectsMap));
		}
<commit_msg>commented analyzer class<commit_after>package mind;

import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.apache.commons.configuration.ConfigurationException;
import org.apache.log4j.Logger;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.InvalidRemoteException;
import org.eclipse.jgit.api.errors.NoHeadException;
import org.eclipse.jgit.api.errors.TransportException;

import com.taskadapter.redmineapi.RedmineException;


/**
 * This class is using a IssueTrackerReader a SCMReader and a SonarReader and
 * creates a table with the violations, number loc, number loc Touched and number defects per class per version (example)
 * Version1={class1={rule1Violations=5;numberDefects=2;size=500;numberLOCTouched=200}}
 * Version2={class1={rule1Violations=2;numberDefects=0;size=550;numberLOCTouched=150}}
 *
 */
public class Analyzer {
	private SonarReader sonarReader;
	private IssueTrackerReader issueTrackerReader;
	private SCMReader scmReader;
	private VersionDAO versionDao;
	private SonarRunnerApi sonarRunner;
	private static org.apache.log4j.Logger log = Logger
			.getLogger(Analyzer.class);

	/**
	 * Initializes the Analyzer and validates the software versions of the different tools 
	 * @param sonarReader
	 * @param issueTrackerReader
	 * @param scmReader
	 * @param sonarRunner
	 * @throws IOException
	 * @throws ConfiguredVersionNotExistInSonarException
	 * @throws UnequalNumberOfVersionsException
	 */
	public Analyzer(SonarReader sonarReader,
			IssueTrackerReader issueTrackerReader, SCMReader scmReader,
			SonarRunnerApi sonarRunner) throws IOException,
			ConfiguredVersionNotExistInSonarException,
			UnequalNumberOfVersionsException {
		this.sonarReader = sonarReader;
		this.issueTrackerReader = issueTrackerReader;
		this.scmReader = scmReader;
		versionDao = new VersionDAO(scmReader, issueTrackerReader, sonarReader);
		this.sonarRunner = sonarRunner;
	}

	/**
	 * Main algorithm to generate the table utilizing the given apis
	 * @return
	 * @throws ConfigurationException
	 * @throws IOException
	 * @throws InvalidRemoteException
	 * @throws TransportException
	 * @throws GitAPIException
	 * @throws RedmineException
	 * @throws VersionIdentifierConflictException
	 * @throws ConfiguredVersionNotExistInSonarException
	 * @throws UnequalNumberOfVersionsException
	 * @throws KeyNotFoundException
	 */
	public LinkedHashMap<String, HashMap<String, HashMap<String, Integer>>> getTechnicalDebtTable()
			throws ConfigurationException, IOException, InvalidRemoteException,
			TransportException, GitAPIException, RedmineException, VersionIdentifierConflictException, ConfiguredVersionNotExistInSonarException, UnequalNumberOfVersionsException, KeyNotFoundException {
		
		LinkedHashMap<String, HashMap<String, HashMap<String, Integer>>> table = new LinkedHashMap<String, HashMap<String, HashMap<String, Integer>>>();
		
		String previousVersionKey = "0";
		HashMap<String, HashMap<String, Integer>> sizeOfResourcePerVersion = new HashMap<String, HashMap<String,Integer>>();
		HashMap<String, HashMap<String, HashMap<String, Integer>>> violationsOfResourcePerVersion = new HashMap<String, HashMap<String,HashMap<String,Integer>>>();
		
		
		for(String currentVersionKey : versionDao.getKeySet())
		{
			sonarRunner.runSonar(versionDao.getScmVersion(currentVersionKey));
			List<String> resources = sonarReader.getListOfAllResources();
			
			HashMap<String,HashMap<String, Integer>> mapOfNumberOfDefectsRelatedToClassPerVersion = getMapOfNumberOfDefectsRelatedToResource(resources, scmReader.getHeadBranch());
			HashMap<String, HashMap<String, Integer>> resourceRows = new HashMap<String, HashMap<String,Integer>>();
			HashMap<String, Integer> sizeOfResources = new HashMap<String, Integer>();
			HashMap<String, HashMap<String, Integer>> violationsOfResource = new HashMap<String, HashMap<String, Integer>>();
			
			
			for (String resource : resources)
			{
				int numberOfDefectsForThisResourceInThisVersion = mapOfNumberOfDefectsRelatedToClassPerVersion.get(versionDao.getMainKeyVersion(currentVersionKey)).get(resource);
				HashMap<String, Integer> technicalDebtRow = new HashMap<String, Integer>();
				
				Integer numberOfLOCTouched;
				int sizeOfClass = sonarReader.getSizeOfClass(resource);
				Iterator<Map.Entry<String, Integer>> it;
				if(previousVersionKey.equals("0") || !violationsOfResourcePerVersion.get(previousVersionKey).containsKey(resource))
				{
					numberOfLOCTouched = null;
					technicalDebtRow.put("size", 0);
					violationsOfResource.put(resource, sonarReader.getNumberOfViolationsPerRuleEverythingZero());
					it = violationsOfResource.get(resource).entrySet().iterator();
				}else
				{
					numberOfLOCTouched = scmReader.getNumberOfLOCtouched(versionDao.getScmVersion(currentVersionKey), versionDao.getScmVersion(previousVersionKey), resource);
					technicalDebtRow.put("size", sizeOfResourcePerVersion.get(previousVersionKey).get(resource));
					it = violationsOfResourcePerVersion.get(previousVersionKey).get(resource).entrySet().iterator();

				}
		 		while (it.hasNext()) {
		 			Map.Entry<String, Integer> pairs = (Map.Entry<String, Integer>) it.next();
				 			technicalDebtRow.put(pairs.getKey(), pairs.getValue());
				 			it.remove();
				 }
				
				technicalDebtRow.put("numberDefects", numberOfDefectsForThisResourceInThisVersion);
				technicalDebtRow.put("locTouched", numberOfLOCTouched);

				resourceRows.put(resource, technicalDebtRow);
				sizeOfResources.put(resource, sizeOfClass);
				violationsOfResource.put(resource, sonarReader.getNumberOfViolationsPerRule(resource));
				
				sizeOfResourcePerVersion.put(currentVersionKey, sizeOfResources);
				violationsOfResourcePerVersion.put(currentVersionKey, violationsOfResource);
			}
			table.put(currentVersionKey, resourceRows);
			previousVersionKey = currentVersionKey;
		}
		return table;
	}

	/**
	 * Returns a map of number of defects related to one resource for example
	 * {Version1={class1=5;class2=3},
	 *  Version2={class1=3;class2=4}}
	 *  The information is pulled from the issue tracker. It is assumed that in the issue is documented to which version this defect was injected.
	 *  The issue number will than be searched in the git revisions. If the issue number is found all affected files will be regarded as files 
	 *  of this defect and hence the defect number for this class will be increased.
	 * @param resources
	 * @param branch
	 * @return
	 * @throws NoHeadException
	 * @throws IOException
	 * @throws GitAPIException
	 * @throws RedmineException
	 * @throws VersionIdentifierConflictException
	 * @throws KeyNotFoundException
	 */
	public HashMap<String, HashMap<String, Integer>> getMapOfNumberOfDefectsRelatedToResource(
			List<String> resources, String branch) throws NoHeadException,
			IOException, GitAPIException, RedmineException,
			VersionIdentifierConflictException, KeyNotFoundException {
		HashMap<String, HashMap<String, Set<Integer>>> mapOfDefectsRelatedToResource = getMapOfDefectsRelatedToResource(
				resources, branch);
		HashMap<String, HashMap<String, Integer>> mapOfNumberOfDefectsRelatedToResource = new HashMap<String, HashMap<String, Integer>>();

		for (Entry<String, HashMap<String, Set<Integer>>> map : mapOfDefectsRelatedToResource
				.entrySet()) {
			HashMap<String, Integer> resourceToNumberOfDefects = new HashMap<String, Integer>();
			for (Entry<String, Set<Integer>> resourceCount : map.getValue()
					.entrySet()) {
				resourceToNumberOfDefects.put(resourceCount.getKey(),
						resourceCount.getValue().size());
				mapOfNumberOfDefectsRelatedToResource.put(map.getKey(),
						resourceToNumberOfDefects);
			}

		}
		return mapOfNumberOfDefectsRelatedToResource;
	}

	/**
	 * Returns a map of defects related to one resource for example
	 * {Version1={class1={issueid=1000, issueid=10001};class2={issueid=1000, issueid=10001}},
	 *  Version2={class1={issueid=1003, issueid=10002}}}
	 * @param resources
	 * @param branch
	 * @return
	 * @throws NoHeadException
	 * @throws IOException
	 * @throws GitAPIException
	 * @throws RedmineException
	 * @throws VersionIdentifierConflictException
	 * @throws KeyNotFoundException
	 */
	private HashMap<String, HashMap<String, Set<Integer>>> getMapOfDefectsRelatedToResource(
			List<String> resources, String branch) throws NoHeadException,
			IOException, GitAPIException, RedmineException,
			VersionIdentifierConflictException, KeyNotFoundException {
		HashMap<String, HashMap<String, Set<Integer>>> mapOfDefectsRelatedToResource = new HashMap<String, HashMap<String, Set<Integer>>>();

		for (String version : versionDao.getKeySet()) {
			HashMap<String, Set<Integer>> resourceToDefectsMap = new HashMap<String, Set<Integer>>();
			for (String resource : resources) {
				Set<Integer> setOfDefectIds = new HashSet<Integer>();
				resourceToDefectsMap.put(resource, setOfDefectIds);
			}
			mapOfDefectsRelatedToResource.put(
					versionDao.getMainKeyVersion(version),
					new HashMap<String, Set<Integer>>(resourceToDefectsMap));
		}

{'fraction_non_alphanumeric': 0.09815228174603174, 'fraction_numerical': 0.004588293650793651, 'mean_word_length': 6.219785138764548, 'pattern_counts': {'":': 0, '<': 149, '<?xml version=': 0, '>': 149, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7536057', 'n_tokens_mistral': 5172, 'n_tokens_neox': 4504, 'n_words': 903}