<commit_before>package edu.ntnu.idi.goldfish.preprocessors;

import edu.ntnu.idi.goldfish.StopWatch;
import edu.ntnu.idi.goldfish.configurations.Config;
import edu.ntnu.idi.goldfish.mahout.DBModel;
import edu.ntnu.idi.goldfish.mahout.DBModel.DBRow;

import org.apache.commons.math3.stat.correlation.PearsonsCorrelation;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.model.DataModel;

import com.google.common.base.Stopwatch;

import java.util.List;
import java.util.stream.Collector;
import java.util.stream.Collectors;

public class PreprocessorStat extends Preprocessor{

	private final int THRESHOLD = 3;
	
	public static enum PredictionMethod { LinearRegression, ClosestNeighbor, EqualBins }
	
	public DataModel preprocess(Config config) throws TasteException {
        DBModel model = config.get("model");
        PredictionMethod predictionMethod = config.get("predictionMethod");
        int minTimeOnPage = config.get("minTimeOnPage");
        double correlationLimit = config.get("correlationLimit");
        
		List<DBModel.DBRow> allResults = model.getFeedbackRows();
		List<DBModel.DBRow> results = allResults.stream().filter(row -> row.rating == 0).collect(Collectors.toList());
		for (DBRow r : results) {
			long itemID = r.itemid;
			
			List<DBModel.DBRow> feedbackForItemID = allResults.stream()
					.filter(row -> row.itemid == itemID)
					.filter(row -> row.rating > 0)
					.collect(Collectors.toList());
			
			if(hasImplicit(r.implicitfeedback) && feedbackForItemID.size() >= THRESHOLD){
				
				double[] dependentVariables = new double[feedbackForItemID.size()]; // the explicit ratings to infer
				double[][] independentVariables = new double[feedbackForItemID.size()][]; // the implicit feedback
				double[] implicitFeedback = null;
				int index = 0;
				
				for (DBRow f : feedbackForItemID) {
					dependentVariables[index] = f.rating;
					
					implicitFeedback = new double[f.implicitfeedback.length];
					for (int i = 0; i < implicitFeedback.length; i++) {
						implicitFeedback[i] = f.implicitfeedback[i];
					}
					independentVariables[index] = implicitFeedback;
					index++;
				}
				
				
				int bestCorrelated = getBestCorrelated(dependentVariables, independentVariables);
				double correlation = getCorrelation(dependentVariables, independentVariables, bestCorrelated);
				
				if(correlation > correlationLimit){
					float pseudoRating = 0;
					switch (predictionMethod) {
					case LinearRegression:
						pseudoRating = getPseudoRatingLinearRegression(dependentVariables, independentVariables, bestCorrelated, r.implicitfeedback);
						break;
					case ClosestNeighbor:
						pseudoRating = getPseudoRatingClosestNeighbor(independentVariables, bestCorrelated, r.implicitfeedback);
						break;
					case EqualBins:
						pseudoRating = getPseudoRatingEqualBins(independentVariables, bestCorrelated, r.implicitfeedback, correlation);
						break;
					default:
						pseudoRating = getPseudoRatingLinearRegression(dependentVariables, independentVariables, bestCorrelated, r.implicitfeedback);
						break;
					}
					
					model.setPreference(r.userid, r.itemid, (float) Math.round(pseudoRating));
					pseudoRatings.add(String.format("%d_%d", r.userid, r.itemid));
				}
				else if(timeOnPageFeedback(r.implicitfeedback, minTimeOnPage, 120000)){
					model.setPreference(r.userid, r.itemid, 4);
					pseudoRatings.add(String.format("%d_%d", r.userid, r.itemid));
				}
			}
		}
		
		return model;
	}
	
	public boolean hasImplicit(float[] implicitfeedback){
		for (int i = 0; i < implicitfeedback.length; i++) {
			if(implicitfeedback[i] > 0) return true;<commit_msg>Sysyout commented<commit_after>package edu.ntnu.idi.goldfish.preprocessors;

import edu.ntnu.idi.goldfish.StopWatch;
import edu.ntnu.idi.goldfish.configurations.Config;
import edu.ntnu.idi.goldfish.mahout.DBModel;
import edu.ntnu.idi.goldfish.mahout.DBModel.DBRow;

import org.apache.commons.math3.stat.correlation.PearsonsCorrelation;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.model.DataModel;

import com.google.common.base.Stopwatch;

import java.util.List;
import java.util.stream.Collector;
import java.util.stream.Collectors;

public class PreprocessorStat extends Preprocessor{

	private final int THRESHOLD = 3;
	
	public static enum PredictionMethod { LinearRegression, ClosestNeighbor, EqualBins }
	
	public DataModel preprocess(Config config) throws TasteException {
        DBModel model = config.get("model");
        PredictionMethod predictionMethod = config.get("predictionMethod");
        int minTimeOnPage = config.get("minTimeOnPage");
        double correlationLimit = config.get("correlationLimit");
        
		List<DBModel.DBRow> allResults = model.getFeedbackRows();
		List<DBModel.DBRow> results = allResults.stream().filter(row -> row.rating == 0).collect(Collectors.toList());
		for (DBRow r : results) {
			long itemID = r.itemid;
			
			List<DBModel.DBRow> feedbackForItemID = allResults.stream()
					.filter(row -> row.itemid == itemID)
					.filter(row -> row.rating > 0)
					.collect(Collectors.toList());
			
			if(hasImplicit(r.implicitfeedback) && feedbackForItemID.size() >= THRESHOLD){
				
				double[] dependentVariables = new double[feedbackForItemID.size()]; // the explicit ratings to infer
				double[][] independentVariables = new double[feedbackForItemID.size()][]; // the implicit feedback
				double[] implicitFeedback = null;
				int index = 0;
				
				for (DBRow f : feedbackForItemID) {
					dependentVariables[index] = f.rating;
					
					implicitFeedback = new double[f.implicitfeedback.length];
					for (int i = 0; i < implicitFeedback.length; i++) {
						implicitFeedback[i] = f.implicitfeedback[i];
					}
					independentVariables[index] = implicitFeedback;
					index++;
				}
				
				
				int bestCorrelated = getBestCorrelated(dependentVariables, independentVariables);
				double correlation = getCorrelation(dependentVariables, independentVariables, bestCorrelated);
				
				if(correlation > correlationLimit){
					float pseudoRating = 0;
					switch (predictionMethod) {
					case LinearRegression:
						pseudoRating = getPseudoRatingLinearRegression(dependentVariables, independentVariables, bestCorrelated, r.implicitfeedback);
						break;
					case ClosestNeighbor:
						pseudoRating = getPseudoRatingClosestNeighbor(independentVariables, bestCorrelated, r.implicitfeedback);
						break;
					case EqualBins:
						pseudoRating = getPseudoRatingEqualBins(independentVariables, bestCorrelated, r.implicitfeedback, correlation);
						break;
					default:
						pseudoRating = getPseudoRatingLinearRegression(dependentVariables, independentVariables, bestCorrelated, r.implicitfeedback);
						break;
					}
					
					model.setPreference(r.userid, r.itemid, (float) Math.round(pseudoRating));
					pseudoRatings.add(String.format("%d_%d", r.userid, r.itemid));
					
//					System.out.println(String.format("%d, %d, %.0f", r.userid, r.itemid, pseudoRating));
				}
				else if(timeOnPageFeedback(r.implicitfeedback, minTimeOnPage, 120000)){
					model.setPreference(r.userid, r.itemid, 4);
					pseudoRatings.add(String.format("%d_%d", r.userid, r.itemid));
				}
			}
		}
		
		return model;
	}
	
	public boolean hasImplicit(float[] implicitfeedback){
		for (int i = 0; i < implicitfeedback.length; i++) {
			if(implicitfeedback[i] > 0) return true;
{'fraction_non_alphanumeric': 0.1118421052631579, 'fraction_numerical': 0.004430719656283566, 'mean_word_length': 4.119587628865979, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5010801', 'n_tokens_mistral': 2791, 'n_tokens_neox': 2249, 'n_words': 426}