<commit_before>/*
 * MSR Tools - tools for mining software repositories
 * 
 * Copyright (C) 2011  <NAME>
 */

using System;
using System.Collections.Generic;
using System.Linq;

using MathNet.Numerics.Statistics;

using MSR.Data;
using MSR.Data.Entities;
using MSR.Data.Entities.DSL.Selection;
using MSR.Data.Entities.DSL.Selection.Metrics;
using MSR.Models.Regressions;

namespace MSR.Models.Prediction.PostReleaseDefectFiles
{
	public abstract class PostReleaseDefectFilesPrediction : Prediction
	{
		public event Action<PostReleaseDefectFilesPrediction,double> CallBack;
		
		protected double defaultCutOffValue;
		private Dictionary<string,double> possibleDefectFiles;
		private IEnumerable<string> defectFiles;
		
		public PostReleaseDefectFilesPrediction()
		{
			defaultCutOffValue = 0.5;
			UseFaultProneProbabilityMeanAsCutOffValue = false;
		}
		public override void Init(IRepositoryResolver repositories, IEnumerable<string> releases)
		{
			base.Init(repositories, releases);
			
			defectFiles = null;
		}
		public virtual void Predict()
		{
			AllFiles = GetFilesInRevision(PredictionRelease);
			
			int allFilesCount = AllFiles.Count();
			int processedFilesCount = 0;
			
			possibleDefectFiles = new Dictionary<string,double>();
			foreach (var file in AllFiles)
			{
				possibleDefectFiles.Add(
					file.Path,
					FileFaultProneProbability(file)
				);
				
				if (CallBack != null)
				{
					processedFilesCount++;
					CallBack(this, (double)processedFilesCount / allFilesCount);
				}
			}
			
			FaultProneProbabilityMean = possibleDefectFiles.Values.Mean();
			
			PredictedDefectFiles = possibleDefectFiles
				.Where(x => x.Value >= CutOffValue)
				.Select(x => x.Key)
				.ToArray();
		}
		public EvaluationResult Evaluate()
		{
			return Evaluate(PredictedDefectFiles);
		}
		public ROCEvaluationResult EvaluateUsingROC()
		{
			List<EvaluationResult> results = new List<EvaluationResult>(101);
			
			for (int cutOffValue = 0; cutOffValue <= 100; cutOffValue++)
			{
				var predictedDefectFiles = possibleDefectFiles
					.Where(x => x.Value >= (double)cutOffValue * 0.01)
					.Select(x => x.Key)
					.ToArray();
				
				results.Add(Evaluate(predictedDefectFiles));
			}
			
			return new ROCEvaluationResult(results.ToArray());
		}
		public bool UseFaultProneProbabilityMeanAsCutOffValue
		{
			get; set;
		}
		
		public double FaultProneProbabilityMean
		{
			get; private set;
		}<commit_msg>ability to set file portion limit for prediction model<commit_after>/*
 * MSR Tools - tools for mining software repositories
 * 
 * Copyright (C) 2011  <NAME>
 */

using System;
using System.Collections.Generic;
using System.Linq;

using MathNet.Numerics.Statistics;

using MSR.Data;
using MSR.Data.Entities;
using MSR.Data.Entities.DSL.Selection;
using MSR.Data.Entities.DSL.Selection.Metrics;
using MSR.Models.Regressions;

namespace MSR.Models.Prediction.PostReleaseDefectFiles
{
	public abstract class PostReleaseDefectFilesPrediction : Prediction
	{
		public event Action<PostReleaseDefectFilesPrediction,double> CallBack;
		
		protected double defaultCutOffValue;
		private Dictionary<string,double> possibleDefectFiles;
		private IEnumerable<string> defectFiles;
		
		public PostReleaseDefectFilesPrediction()
		{
			defaultCutOffValue = 0.5;
			UseFaultProneProbabilityMeanAsCutOffValue = false;
			FilePortionLimit = 1;
		}
		public override void Init(IRepositoryResolver repositories, IEnumerable<string> releases)
		{
			base.Init(repositories, releases);
			
			defectFiles = null;
		}
		public virtual void Predict()
		{
			AllFiles = GetFilesInRevision(PredictionRelease);
			
			int allFilesCount = AllFiles.Count();
			int processedFilesCount = 0;
			
			possibleDefectFiles = new Dictionary<string,double>();
			foreach (var file in AllFiles)
			{
				possibleDefectFiles.Add(
					file.Path,
					FileFaultProneProbability(file)
				);
				
				if (CallBack != null)
				{
					processedFilesCount++;
					CallBack(this, (double)processedFilesCount / allFilesCount);
				}
			}
			
			FaultProneProbabilityMean = possibleDefectFiles.Values.Mean();
			
			PredictedDefectFiles = possibleDefectFiles
				.Where(x => x.Value >= CutOffValue)
				.TakeNoMoreThan((int)(allFilesCount * FilePortionLimit))
				.Select(x => x.Key)
				.ToArray();
		}
		public EvaluationResult Evaluate()
		{
			return Evaluate(PredictedDefectFiles);
		}
		public ROCEvaluationResult EvaluateUsingROC()
		{
			List<EvaluationResult> results = new List<EvaluationResult>(101);
			
			for (int cutOffValue = 0; cutOffValue <= 100; cutOffValue++)
			{
				var predictedDefectFiles = possibleDefectFiles
					.Where(x => x.Value >= (double)cutOffValue * 0.01)
					.Select(x => x.Key)
					.ToArray();
				
				results.Add(Evaluate(predictedDefectFiles));
			}
			
			return new ROCEvaluationResult(results.ToArray());
		}
		public bool UseFaultProneProbabilityMeanAsCutOffValue
		{
			get; set;
		}
		public double FilePortionLimit
		{
			get; set;
		}
		
		public double FaultProneProbabilityMean
		{
			get; private set;
		}
{'fraction_non_alphanumeric': 0.0946347257968719, 'fraction_numerical': 0.006929320926549199, 'mean_word_length': 3.9191820837390456, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1223492', 'n_tokens_mistral': 1958, 'n_tokens_neox': 1539, 'n_words': 319}