<commit_before>package edu.ntnu.idi.goldfish.preprocessors;

import edu.ntnu.idi.goldfish.mahout.SMDataModel;
import edu.ntnu.idi.goldfish.mahout.SMPreference;

import org.apache.commons.math3.exception.NumberIsTooSmallException;
import org.apache.commons.math3.stat.correlation.PearsonsCorrelation;
import org.apache.commons.math3.stat.regression.OLSMultipleLinearRegression;
import org.apache.mahout.cf.taste.common.NoSuchItemException;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.model.Preference;
import org.apache.mahout.cf.taste.model.PreferenceArray;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * This class is used to map implicit feedback to explicit rating where it is possible.
 * @author <NAME> and <NAME>
 *
 */
public class PreprocessorMLR extends Preprocessor {

	private final int THRESHOLD = 3;
	private final int TIME_ON_PAGE_INDEX = 1;
	private final int TIME_ON_MOUSE_INDEX = 2;
	private final int RATING_INDEX = 0;
	
	private Map<String, Float> correlations = new HashMap<String, Float>();
	private static Set<String> pseudoRatings = new HashSet<String>();


	public static DataModel getPreprocessedDataModel(String path) throws Exception {
		YowModel model;
		model = new YowModel(new File(path));
		PreprocessorMLR pre = new PreprocessorMLR();
		pre.preprocess(model);
		return model;
	}

	public static boolean isPseudoPreference(Preference pref) {
		return pseudoRatings.contains(String.format("%d_%d", pref.getUserID(), pref.getItemID()));
	}

	public void preprocess(YowModel model) {
		try {
			
			double[] beta = globalLR(model, 2);
			
			LongPrimitiveIterator it = model.getItemIDs();
			while(it.hasNext()){
				long itemID = it.next();
				PreferenceArray prefs = model.getPreferencesForItem(itemID);
				
				for (Preference p : prefs) {
					
					SMPreference pref = (SMPreference) p;
					boolean hasExplicit = pref.getValue(RATING_INDEX) >= 1;
					if(hasExplicit){
						// No need to add a pseudo rating
						continue;
					}
					
					float[] feedback = pref.getValues();
					
					float pseudoRating = (float) beta[0];
					for (int i = 1; i < beta.length; i++) {
						pseudoRating += beta[i]*feedback[i];
					}
					pref.setValue(Math.round(pseudoRating), RATING_INDEX);
					pseudoRatings.add(String.format("%d_%d", pref.getUserID(), pref.getItemID()));
				}
			}
		} catch (TasteException e) {
			e.printStackTrace();
			System.err.println("Could not get all items from the yow model");
		}
		
	}
	
	private double[] globalLR(YowModel model, int numberOfIndependentVariables) throws TasteException {
		
		if(numberOfIndependentVariables == 0) throw new NumberIsTooSmallException(numberOfIndependentVariables, 1, true);
		
		List<Double> tempExpl = new ArrayList<Double>();<commit_msg>Fix build error<commit_after>package edu.ntnu.idi.goldfish.preprocessors;

import edu.ntnu.idi.goldfish.mahout.SMDataModel;
import edu.ntnu.idi.goldfish.mahout.SMPreference;

import org.apache.commons.math3.exception.NumberIsTooSmallException;
import org.apache.commons.math3.stat.correlation.PearsonsCorrelation;
import org.apache.commons.math3.stat.regression.OLSMultipleLinearRegression;
import org.apache.mahout.cf.taste.common.NoSuchItemException;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.model.Preference;
import org.apache.mahout.cf.taste.model.PreferenceArray;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * This class is used to map implicit feedback to explicit rating where it is possible.
 * @author <NAME> and <NAME>
 *
 */
public class PreprocessorMLR extends Preprocessor {

	private final int THRESHOLD = 3;
	private final int TIME_ON_PAGE_INDEX = 1;
	private final int TIME_ON_MOUSE_INDEX = 2;
	private final int RATING_INDEX = 0;
	
	private Map<String, Float> correlations = new HashMap<String, Float>();


	public static DataModel getPreprocessedDataModel(String path) throws Exception {
		YowModel model;
		model = new YowModel(new File(path));
		PreprocessorMLR pre = new PreprocessorMLR();
		pre.preprocess(model);
		return model;
	}


	public DataModel preprocess(YowModel model) {
		try {
			
			double[] beta = globalLR(model, 2);
			
			LongPrimitiveIterator it = model.getItemIDs();
			while(it.hasNext()){
				long itemID = it.next();
				PreferenceArray prefs = model.getPreferencesForItem(itemID);
				
				for (Preference p : prefs) {
					
					SMPreference pref = (SMPreference) p;
					boolean hasExplicit = pref.getValue(RATING_INDEX) >= 1;
					if(hasExplicit){
						// No need to add a pseudo rating
						continue;
					}
					
					float[] feedback = pref.getValues();
					
					float pseudoRating = (float) beta[0];
					for (int i = 1; i < beta.length; i++) {
						pseudoRating += beta[i]*feedback[i];
					}
					pref.setValue(Math.round(pseudoRating), RATING_INDEX);
					pseudoRatings.add(String.format("%d_%d", pref.getUserID(), pref.getItemID()));
				}
			}
		} catch (TasteException e) {
			e.printStackTrace();
			System.err.println("Could not get all items from the yow model");
		}
        return null;
		
	}
	
	private double[] globalLR(YowModel model, int numberOfIndependentVariables) throws TasteException {
		
		if(numberOfIndependentVariables == 0) throw new NumberIsTooSmallException(numberOfIndependentVariables, 1, true);
		
		List<Double> tempExpl = new ArrayList<Double>();
{'fraction_non_alphanumeric': 0.11167341430499325, 'fraction_numerical': 0.0043859649122807015, 'mean_word_length': 4.27491103202847, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '676323', 'n_tokens_mistral': 2182, 'n_tokens_neox': 1894, 'n_words': 426}