<commit_before>package sw705e15.wi.learning;


import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

import org.jblas.*;

public class Processing {

	// the total amount of ratings
	private static HashMap<Integer, Integer> observedUserRatings = new HashMap<>();
	private static HashMap<Integer, Integer> observedMovieRatings = new HashMap<>();

	// summation of the ratings
	private static HashMap<Integer, Integer> summarizedUserRatings = new HashMap<>();
	private static HashMap<Integer, Integer> summarizedMovieRatings = new HashMap<>();

	private static double observedRatingsOverall = 0;
	private static double summarizedRatingsOverall = 0;

	private static double totalNormalization = 0;
	
	private enum ProcessingTypes{PRE, POST};

	public Processing(final DoubleMatrix input) {
		DoubleMatrix data = new DoubleMatrix(new double[][]{{4, 2, 5}, {3, 1, 4}, {1, 4, 2}});
		DoubleMatrix newData = preProcessing(data, ProcessingTypes.PRE);
		DoubleMatrix newData2 = preProcessing(newData, ProcessingTypes.POST);
	}

	static public DoubleMatrix preProcessing(final DoubleMatrix initialMatrix, final ProcessingTypes procType) {
		
		switch (procType) {
		case PRE:
			
			// Iterate through all rows and count their amount of ratings; Movies
			for (int elementCount = 0; elementCount < initialMatrix.getRows(); elementCount++) {
				for (int columnCount = 0; columnCount < initialMatrix.getColumns(); columnCount++) {
					
					List<Double> currentRow = initialMatrix.getRow(elementCount).elementsAsList();
					int amountOfRatingsForMovie = 0;
					int summarizedRatingsForMovie = 0;
					for (Double rowElement : currentRow) {
						if (rowElement != 0) {
							observedRatingsOverall++;
							summarizedRatingsOverall += rowElement;
							amountOfRatingsForMovie++;
							summarizedRatingsForMovie += rowElement;
						}
					}
					
					observedMovieRatings.put(elementCount, amountOfRatingsForMovie);
					summarizedMovieRatings.put(elementCount, summarizedRatingsForMovie);
				}
			}

			// Iterate through all columns and count their amount of ratings; Users
			for (int elementCount = 0; elementCount < initialMatrix.getColumns(); elementCount++) {
				List<Double> currentColumn = initialMatrix.getColumn(elementCount).elementsAsList();
				int amountOfRatingsForUser = 0;
				int summarizedRatingsForUser = 0;
				for (Double columnElement : currentColumn) {
					if (columnElement != 0) {
						amountOfRatingsForUser++;
						summarizedRatingsForUser += columnElement;
					}
				}
				observedUserRatings.put(elementCount, amountOfRatingsForUser);
				summarizedUserRatings.put(elementCount, summarizedRatingsForUser);
			}

			totalNormalization = (1.0 / observedRatingsOverall) * summarizedRatingsOverall;

			for (int rowCount = 0; rowCount < initialMatrix.getRows(); rowCount++) {

				double movieNormalization = (1.0 / observedMovieRatings.get(rowCount)) * summarizedMovieRatings.get(rowCount);
				for (int columnCount = 0; columnCount < initialMatrix.getColumns(); columnCount++) {
					double initialValue = initialMatrix.get(rowCount, columnCount);
					double userNormalization = (1.0 / observedUserRatings.get(columnCount))
							* summarizedUserRatings.get(columnCount);
					double newValue = initialValue - movieNormalization - userNormalization + totalNormalization;
					initialMatrix.put(rowCount, columnCount, newValue);
				}
			}

			return initialMatrix;
			
		case POST:
			
			for (int rowCount = 0; rowCount < initialMatrix.getRows(); rowCount++) {

				double movieNormalization = (1.0 / observedMovieRatings.get(rowCount)) * summarizedMovieRatings.get(rowCount);
				for (int columnCount = 0; columnCount < initialMatrix.getColumns(); columnCount++) {
					double initialValue = initialMatrix.get(rowCount, columnCount);
					double userNormalization = (1.0 / observedUserRatings.get(columnCount))
							* summarizedUserRatings.get(columnCount);
					double newValue = initialValue + movieNormalization + userNormalization - totalNormalization;
					initialMatrix.put(rowCount, columnCount, newValue);
				}
			}

			return initialMatrix;
			
		default:
			System.out.println("Did not understand processingtype enum value, returning null.");
			return null;
		}
		
	}
	
	
	public DoubleMatrix funkSVD(final DoubleMatrix data)
	{
		return data;
	}

}<commit_msg>more implementation in processing<commit_after>package sw705e15.wi.learning;


import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;

import org.jblas.*;
import org.jblas.util.Random;

import sw705e15.wi.parsing.SparseMatrix;

public class Processing {

	// the total amount of ratings
	private static final HashMap<Integer, Integer> observedUserRatings = new HashMap<>();
	private static final HashMap<Integer, Integer> observedMovieRatings = new HashMap<>();

	// summation of the ratings
	private static final HashMap<Integer, Integer> summarizedUserRatings = new HashMap<>();
	private static final HashMap<Integer, Integer> summarizedMovieRatings = new HashMap<>();

	private static double observedRatingsOverall = 0;
	private static double summarizedRatingsOverall = 0;
	private static double totalNormalization = 0;
	
	private enum ProcessingTypes{PRE, POST};

	private static final int K = 35;
	private static final double SCALE_CONSTANT = 0.001;
	
	private static SparseMatrix A;
	private static SparseMatrix B;
	
	
	public Processing(final SparseMatrix input) {
		//;
		SparseMatrix newData = preOrPostProcessing(input, ProcessingTypes.PRE);
		
		SparseMatrix dataengangtil = funkSVD(newData);
		
		
	}

	static public SparseMatrix preOrPostProcessing(final SparseMatrix initialMatrix, final ProcessingTypes procType) {
		
		switch (procType) {
		case PRE:
			
			// Iterate through all rows and count their amount of ratings; Movies
			for (int rowCount = 0; rowCount < initialMatrix.rowSize(); rowCount++) {
				
				int amountOfRatingsForMovie = 0;
				int summarizedRatingsForMovie = 0;
				
				for (int columnCount = 0; columnCount < initialMatrix.columnSize(); columnCount++) {
					
					double currentVal = initialMatrix.get(rowCount, columnCount);
					observedRatingsOverall++;
					summarizedRatingsOverall += currentVal;
					amountOfRatingsForMovie++;
					summarizedRatingsForMovie += currentVal;
				}
					
					observedMovieRatings.put(rowCount, amountOfRatingsForMovie);
					summarizedMovieRatings.put(rowCount, summarizedRatingsForMovie);
			}

			
			for (int columnCount = 0; columnCount < initialMatrix.columnSize(); columnCount++) {
				
				int amountOfRatingsForUser = 0;
				int summarizedRatingsForUser = 0;
				
				for (int rowCount = 0; rowCount < initialMatrix.rowSize(); rowCount++) {
					
					double currentVal = initialMatrix.get(rowCount, columnCount);
					observedRatingsOverall++;
					summarizedRatingsOverall += currentVal;
					amountOfRatingsForUser++;
					summarizedRatingsForUser += currentVal;
				}
					
					observedUserRatings.put(columnCount, amountOfRatingsForUser);
					summarizedUserRatings.put(columnCount, summarizedRatingsForUser);
			}

			totalNormalization = (1.0 / observedRatingsOverall) * summarizedRatingsOverall;

			for (int rowCount = 0; rowCount < initialMatrix.rowSize(); rowCount++) {

				double movieNormalization = (1.0 / observedMovieRatings.get(rowCount)) * summarizedMovieRatings.get(rowCount);
				
				for (int columnCount = 0; columnCount < initialMatrix.columnSize(); columnCount++) {
					double initialValue = initialMatrix.get(rowCount, columnCount);
					double userNormalization = (1.0 / observedUserRatings.get(columnCount)) * summarizedUserRatings.get(columnCount);
					double newValue = initialValue - movieNormalization - userNormalization + totalNormalization;
					initialMatrix.set(rowCount, columnCount, newValue);
				}
			}

			return initialMatrix;
			
		case POST:
			
			for (int rowCount = 0; rowCount < initialMatrix.rowSize(); rowCount++) {

				double movieNormalization = (1.0 / observedMovieRatings.get(rowCount)) * summarizedMovieRatings.get(rowCount);
				for (int columnCount = 0; columnCount < initialMatrix.columnSize(); columnCount++) {
					double initialValue = initialMatrix.get(rowCount, columnCount);
					double userNormalization = (1.0 / observedUserRatings.get(columnCount))
							* summarizedUserRatings.get(columnCount);
					double newValue = initialValue + movieNormalization + userNormalization - totalNormalization;
					initialMatrix.set(rowCount, columnCount, newValue);
				}
			}

			return initialMatrix;
			
		default:
			System.out.println("Did not understand processingtype enum value, returning null.");
			return null;
		}
		
	}
	
	
	public SparseMatrix funkSVD(final SparseMatrix data)
	{
		A = new SparseMatrix(data.rowSize(), K);
		B = new SparseMatrix(K, data.columnSize());
		
		for (int rowCount = 0; rowCount < A.rowSize(); rowCount++) {
			for (int colCount = 0; colCount < A.columnSize(); colCount++) {
				A.set(rowCount, colCount, Math.random());
			}
		}
		
		for (int rowCount = 0; rowCount < B.rowSize(); rowCount++) {
			for (int colCount = 0; colCount < B.columnSize(); colCount++) {
				B.set(rowCount, colCount, Math.random());
			}
		}
		
		// SKIFT DET HER LOOP NR VI VED HVOR LANG TID VI SKAL RENDE
		while(true)
		{
			stochasticGradientDescentStepForA(data);
			stochasticGradientDescentStepForB(data);
		}
	}
	
	private SparseMatrix stochasticGradientDescentStepForA(final SparseMatrix data)
	{
		int randomMovieIndex = Random.nextInt(A.rowSize());
		int randomUserIndex = Random.nextInt(B.columnSize());
		
		double originalValue = data.get(randomMovieIndex, randomUserIndex);
		double[] addend = new double[K]; 
		
		for (int count = 0; count < K; count++) {
			
			double currentAValue = A.get(randomMovieIndex, count);
			double currentBValue = B.get(count, randomUserIndex);
			
			double sum = 0;
			
			for (int i = 0; i < K; i++) {
				double temporaryAValue = A.get(randomMovieIndex, i);
				double temporaryBValue = B.get(i, randomUserIndex);
				sum += temporaryAValue * temporaryBValue;
			}
			addend[count] = currentAValue + SCALE_CONSTANT * (originalValue - sum) * currentBValue;
		}
		
		for (int addendCount = 0; addendCount < addend.length; addendCount++) {
			A.set(randomMovieIndex, addendCount, addend[addendCount]);
		}
		
		return A;
	}
	
	private SparseMatrix stochasticGradientDescentStepForB(final SparseMatrix data)
	{
		int randomMovieIndex = Random.nextInt(A.rowSize());
		int randomUserIndex = Random.nextInt(B.columnSize());
		
		double originalValue = data.get(randomMovieIndex, randomUserIndex);
		double[] addend = new double[K]; 
		
		for (int count = 0; count < K; count++) {
			
			double currentAValue = A.get(randomMovieIndex, count);
			double currentBValue = B.get(count, randomUserIndex);
			
			double sum = 0;
			
			for (int i = 0; i < K; i++) {
				double temporaryAValue = A.get(randomMovieIndex, i);
				double temporaryBValue = B.get(i, randomUserIndex);
				sum += temporaryAValue * temporaryBValue;
			}
			addend[count] = currentBValue + SCALE_CONSTANT * (originalValue - sum) * currentAValue;
		}
		
		for (int addendCount = 0; addendCount < addend.length; addendCount++) {
			B.set(randomMovieIndex, addendCount, addend[addendCount]);
		}
		
		return B;
	}
	
	private HashMap<Integer, Double> prediction(final SparseMatrix initialMatrix, final int userID, final List<Integer> movieIDs)
	{
		HashMap<Integer, Double> userMoviePredictions = new HashMap<>();
		for (Integer movieID : movieIDs) {
			
			double predictionValue = 0;
			
			for (int k = 0; k < K; k++) {
				predictionValue += A.get(movieID, k) * B.get(k, userID);
			}
			userMoviePredictions.put(movieID, predictionValue);
		}
		
		double userNormalization = (1.0 / observedUserRatings.get(userID)) * summarizedUserRatings.get(userID);
		for (Entry<Integer, Double> mapEntry : userMoviePredictions.entrySet()) {
			double movieNormalization = (1.0 / observedMovieRatings.get(mapEntry.getKey())) * summarizedMovieRatings.get(mapEntry.getKey());
			userMoviePredictions.put(mapEntry.getKey(), mapEntry.getValue() + userNormalization + movieNormalization - totalNormalization);
		}

		return userMoviePredictions;	
	}

}
{'fraction_non_alphanumeric': 0.10258096334045701, 'fraction_numerical': 0.00821962847279303, 'mean_word_length': 4.2107066381156315, 'pattern_counts': {'":': 0, '<': 52, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5382886', 'n_tokens_mistral': 4432, 'n_tokens_neox': 3471, 'n_words': 868}