<commit_before>package org.pescuma.buildhealth.analyser.staticanalysis;

import static java.util.Arrays.*;
import static org.pescuma.buildhealth.analyser.utils.NumbersFormater.*;
import static org.pescuma.buildhealth.core.prefs.BuildHealthPreference.*;
import static org.pescuma.buildhealth.utils.StringHelper.*;

import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Deque;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.kohsuke.MetaInfServices;
import org.pescuma.buildhealth.analyser.BuildHealthAnalyser;
import org.pescuma.buildhealth.analyser.utils.BuildHealthAnalyserUtils.TreeStats;
import org.pescuma.buildhealth.analyser.utils.BuildStatusAndExplanation;
import org.pescuma.buildhealth.analyser.utils.BuildStatusFromThresholdComputer;
import org.pescuma.buildhealth.analyser.utils.SimpleTree;
import org.pescuma.buildhealth.core.BuildData;
import org.pescuma.buildhealth.core.BuildData.Line;
import org.pescuma.buildhealth.core.BuildStatus;
import org.pescuma.buildhealth.core.Report;
import org.pescuma.buildhealth.core.prefs.BuildHealthPreference;
import org.pescuma.buildhealth.prefs.Preferences;
import org.pescuma.buildhealth.utils.Location;

import com.google.common.base.Function;

/**
 * Expect the lines to be:
 * 
 * <pre>
 * Static analysis,language,framework,location,category,message,{severity:Low,Medium,High},details,URL with details
 * </pre>
 * 
 * The number is the number of violations in that place.
 * 
 * Location can be one of:
 * <ul>
 * <li>filename
 * <li>filename>line
 * <li>filename>line:column
 * <li>filename>beginLine:beginColumn:endLine:endColumn
 * <li>or multiple of above, separated by | (ex: file1|file2:32)
 * </ul>
 * 
 * To be able to use the details (anything after filename) at least the filename is required.
 * 
 * Example:
 * 
 * <pre>
 * 1 | Static analysis,Java,Task,/a/b.java,12,Type1,Go to bed
 * 1 | Static analysis,Java,PMD,/a/b.java,12:1:12:5,A/Type1,Go to bed,http://a.com/info.html
 * 10 | Static analysis,Java,Task,/a/c.java,,Type2
 * </pre>
 */
@MetaInfServices
public class StaticAnalysisAnalyser implements BuildHealthAnalyser {
	
	public static final int COLUMN_LANGUAGE = 1;
	public static final int COLUMN_FRAMEWORK = 2;
	public static final int COLUMN_LOCATION = 3;
	public static final int COLUMN_CATEGORY = 4;
	public static final int COLUMN_MESSAGE = 5;
	public static final int COLUMN_SEVERITY = 6;
	public static final int COLUMN_DETAILS = 7;
	public static final int COLUMN_URL = 8;
	
	private static final BuildStatusFromThresholdComputer statusComputer = new BuildStatusFromThresholdComputer(false) {
		@Override
		protected String computeSoSoMessage(double good, String[] prefKey) {
			return "Instable if has more than " + formatValue(good) + " violations"
					+ getPrefKeyDetails(asList(prefKey));
		}
		
		@Override
		protected String computeProblematicMessage(double warn, String[] prefKey) {
			return "Should not have more than " + formatValue(warn) + " violations"
					+ getPrefKeyDetails(asList(prefKey));
		}
	};
	
	@Override
	public String getName() {
		return "Static analysis";
	}
	
	@Override
	public int getPriority() {
		return 300;
	}
	
	@Override
	public List<BuildHealthPreference> getPreferences() {
		List<BuildHealthPreference> result = new ArrayList<BuildHealthPreference>();
		
		result.add(new BuildHealthPreference("Maximun munber of violations for a Good build", "<no limit>",
				"staticanalysis", "good"));
		result.add(new BuildHealthPreference("Maximun munber of violations for a So So build", "<no limit>",
				"staticanalysis", "warn"));
		
		result.add(new BuildHealthPreference("Maximun munber of violations for a Good build", "<no limit>",
				"staticanalysis", ANY_VALUE_KEY_PREFIX + "<language>", "good"));
		result.add(new BuildHealthPreference("Maximun munber of violations for a So So build", "<no limit>",
				"staticanalysis", ANY_VALUE_KEY_PREFIX + "<language>", "warn"));
		
		result.add(new BuildHealthPreference("Maximun munber of violations for a Good build", "<no limit>",
				"staticanalysis", ANY_VALUE_KEY_PREFIX + "<language>", ANY_VALUE_KEY_PREFIX + "<framework>", "good"));
		result.add(new BuildHealthPreference("Maximun munber of violations for a So So build", "<no limit>",
				"staticanalysis", ANY_VALUE_KEY_PREFIX + "<language>", ANY_VALUE_KEY_PREFIX + "<framework>", "warn"));
		
		return Collections.unmodifiableList(result);
	}
	
	@Override
	public List<Report> computeReport(BuildData data, Preferences prefs, int opts) {
		data = data.filter("Static analysis");
		if (data.isEmpty())
			return Collections.emptyList();
		
		prefs = prefs.child("staticanalysis");
		
		SimpleTree<Stats> tree = buildTree(data);
		
		sumChildStatsAndComputeBuildStatuses(tree, prefs);
		
		return asList(toReport(tree.getRoot(), getName(), prefs, 1));
	}
	
	private SimpleTree<Stats> buildTree(BuildData data) {
		SimpleTree<Stats> tree = new SimpleTree<Stats>(new Function<String[], Stats>() {
			@Override
			public Stats apply(String[] name) {
				return new Stats(name);
			}
		});
		
		for (Line line : data.getLines()) {
			SimpleTree<Stats>.Node node = tree.getRoot();
			
			node = node.getChild(getLanguage(line));
			node = node.getChild(line.getColumn(COLUMN_FRAMEWORK));
			
			String category = line.getColumn(COLUMN_CATEGORY);
			if (!category.isEmpty())
				node = node.getChild(category);
			
			Stats stats = node.getData();<commit_msg>Static analysis analyser: better handling of categories<commit_after>package org.pescuma.buildhealth.analyser.staticanalysis;

import static java.util.Arrays.*;
import static org.pescuma.buildhealth.analyser.utils.NumbersFormater.*;
import static org.pescuma.buildhealth.core.prefs.BuildHealthPreference.*;
import static org.pescuma.buildhealth.utils.StringHelper.*;

import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Deque;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.kohsuke.MetaInfServices;
import org.pescuma.buildhealth.analyser.BuildHealthAnalyser;
import org.pescuma.buildhealth.analyser.utils.BuildHealthAnalyserUtils.TreeStats;
import org.pescuma.buildhealth.analyser.utils.BuildStatusAndExplanation;
import org.pescuma.buildhealth.analyser.utils.BuildStatusFromThresholdComputer;
import org.pescuma.buildhealth.analyser.utils.SimpleTree;
import org.pescuma.buildhealth.core.BuildData;
import org.pescuma.buildhealth.core.BuildData.Line;
import org.pescuma.buildhealth.core.BuildStatus;
import org.pescuma.buildhealth.core.Report;
import org.pescuma.buildhealth.core.prefs.BuildHealthPreference;
import org.pescuma.buildhealth.prefs.Preferences;
import org.pescuma.buildhealth.utils.Location;

import com.google.common.base.Function;

/**
 * Expect the lines to be:
 * 
 * <pre>
 * Static analysis,language,framework,location,category,message,{severity:Low,Medium,High},details,URL with details
 * </pre>
 * 
 * The number is the number of violations in that place.
 * 
 * Location can be one of:
 * <ul>
 * <li>filename
 * <li>filename>line
 * <li>filename>line:column
 * <li>filename>beginLine:beginColumn:endLine:endColumn
 * <li>or multiple of above, separated by | (ex: file1|file2:32)
 * </ul>
 * 
 * To be able to use the details (anything after filename) at least the filename is required.
 * 
 * Example:
 * 
 * <pre>
 * 1 | Static analysis,Java,Task,/a/b.java,12,Type1,Go to bed
 * 1 | Static analysis,Java,PMD,/a/b.java,12:1:12:5,A/Type1,Go to bed,http://a.com/info.html
 * 10 | Static analysis,Java,Task,/a/c.java,,Type2
 * </pre>
 */
@MetaInfServices
public class StaticAnalysisAnalyser implements BuildHealthAnalyser {
	
	public static final int COLUMN_LANGUAGE = 1;
	public static final int COLUMN_FRAMEWORK = 2;
	public static final int COLUMN_LOCATION = 3;
	public static final int COLUMN_CATEGORY = 4;
	public static final int COLUMN_MESSAGE = 5;
	public static final int COLUMN_SEVERITY = 6;
	public static final int COLUMN_DETAILS = 7;
	public static final int COLUMN_URL = 8;
	
	private static final BuildStatusFromThresholdComputer statusComputer = new BuildStatusFromThresholdComputer(false) {
		@Override
		protected String computeSoSoMessage(double good, String[] prefKey) {
			if (isZero(good))
				return "Instable if has any violations" + getPrefKeyDetails(asList(prefKey));
			else
				return "Instable if has more than " + formatValue(good) + " violations"
						+ getPrefKeyDetails(asList(prefKey));
		}
		
		@Override
		protected String computeProblematicMessage(double warn, String[] prefKey) {
			if (isZero(warn))
				return "Should have no violations" + getPrefKeyDetails(asList(prefKey));
			else
				return "Should not have more than " + formatValue(warn) + " violations"
						+ getPrefKeyDetails(asList(prefKey));
		}
		
		private boolean isZero(double warn) {
			return (int) warn == 0;
		}
	};
	
	@Override
	public String getName() {
		return "Static analysis";
	}
	
	@Override
	public int getPriority() {
		return 300;
	}
	
	@Override
	public List<BuildHealthPreference> getPreferences() {
		List<BuildHealthPreference> result = new ArrayList<BuildHealthPreference>();
		
		result.add(new BuildHealthPreference("Maximun munber of violations for a Good build", "<no limit>",
				"staticanalysis", "good"));
		result.add(new BuildHealthPreference("Maximun munber of violations for a So So build", "<no limit>",
				"staticanalysis", "warn"));
		
		result.add(new BuildHealthPreference("Maximun munber of violations for a Good build", "<no limit>",
				"staticanalysis", ANY_VALUE_KEY_PREFIX + "<language>", "good"));
		result.add(new BuildHealthPreference("Maximun munber of violations for a So So build", "<no limit>",
				"staticanalysis", ANY_VALUE_KEY_PREFIX + "<language>", "warn"));
		
		result.add(new BuildHealthPreference("Maximun munber of violations for a Good build", "<no limit>",
				"staticanalysis", ANY_VALUE_KEY_PREFIX + "<language>", ANY_VALUE_KEY_PREFIX + "<framework>", "good"));
		result.add(new BuildHealthPreference("Maximun munber of violations for a So So build", "<no limit>",
				"staticanalysis", ANY_VALUE_KEY_PREFIX + "<language>", ANY_VALUE_KEY_PREFIX + "<framework>", "warn"));
		
		return Collections.unmodifiableList(result);
	}
	
	@Override
	public List<Report> computeReport(BuildData data, Preferences prefs, int opts) {
		data = data.filter("Static analysis");
		if (data.isEmpty())
			return Collections.emptyList();
		
		prefs = prefs.child("staticanalysis");
		
		SimpleTree<Stats> tree = buildTree(data);
		
		sumChildStatsAndComputeBuildStatuses(tree, prefs);
		
		return asList(toReport(tree.getRoot(), getName(), prefs, 1));
	}
	
	private SimpleTree<Stats> buildTree(BuildData data) {
		SimpleTree<Stats> tree = new SimpleTree<Stats>(new Function<String[], Stats>() {
			@Override
			public Stats apply(String[] name) {
				return new Stats(name);
			}
		});
		
		for (Line line : data.getLines()) {
			SimpleTree<Stats>.Node node = tree.getRoot();
			
			node = node.getChild(getLanguage(line));
			node = node.getChild(line.getColumn(COLUMN_FRAMEWORK));
			
			String category = line.getColumn(COLUMN_CATEGORY);
			if (!category.isEmpty()) {
				for (String cat : category.split("/|\\\\")) {
					cat = cat.trim();
					if (cat.isEmpty())
						cat = "<no name>";
					node = node.getChild(cat);
				}
			}
			
			Stats stats = node.getData();
{'fraction_non_alphanumeric': 0.12009076627683714, 'fraction_numerical': 0.005498341769942398, 'mean_word_length': 5.040590405904059, 'pattern_counts': {'":': 0, '<': 70, '<?xml version=': 0, '>': 76, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6351512', 'n_tokens_mistral': 3920, 'n_tokens_neox': 3532, 'n_words': 855}