<commit_before>package algorithm;

import controllers.Factory;
import models.core.Account;
import models.core.AnswerRecord;
import models.core.Question;
import models.core.Rule;
import org.hibernate.Session;
import org.hibernate.criterion.Projections;
import play.Logger;
import util.HibernateUtil;

import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Created by ilia on 31.10.14.
 */
public class CollaborativeFiltering {

    private static final Logger.ALogger logger = Logger.of(CollaborativeFiltering.class);

    private static Long level;
    private static Rule rule;
    private static double progress;
    private static double maxProgress = 0.8;
    private static boolean allAnswers = false;
    private static Session session;

    public static Question getNextQuestion(Account account,Long accountLevel, double accountProgress){
        session = HibernateUtil.getSessionFactory().openSession();
        List<AnswerRecord> answerRecords;
        try{
            answerRecords = (ArrayList) Factory.getInstance().getAnswerRecordDAO().getAllAnswerRecords();
        }catch (SQLException ex) {
            logger.error("Error occurred when getting all answerRecords: " + ex.getMessage());
            return null;
        }

//        getCurrentProgress(account,answerRecords);
        level = accountLevel;// account.getAccountLevel();
        if(level == null) level = 1L;
        progress = accountProgress;//account.getLevelProgress();
        if(progress >= maxProgress){
            level++;
            allAnswers = false;
            progress = 0.0;
        }

        //userId->(questionId->rating))
        Map<Long,Map<Long,Double>> userChoices = getChoicesRating(answerRecords);

        //cos measure
        Map<Long,Double> measure = new HashMap<>();
        double measureSum = 0.0;

        double userNorm = 0.0;
        for(Long questionId : userChoices.get(account.id).keySet())
            userNorm += userChoices.get(account.id).get(questionId) * userChoices.get(account.id).get(questionId);
        userNorm = Math.sqrt(userNorm);

        for(Long userId : userChoices.keySet()){
            if(!userId.equals(account.id)){
                double temp = 0.0;  //scalar mult
                double norm = 0.0;
                for(Long questionId : userChoices.get(userId).keySet()){
                    temp += userChoices.get(userId).get(questionId) * userChoices.get(account.id).get(questionId);
                    norm += userChoices.get(userId).get(questionId) * userChoices.get(userId).get(questionId);
                }
                norm = Math.sqrt(norm);
                if(norm != 0.0){
                    measure.put(userId,temp / (norm * userNorm));
                    measureSum += temp / (norm * userNorm);
                }
                else measure.put(userId,0.0);
            }
        }

        if(measureSum == 0.0){
            logger.error("MeasureSum is ZERO");
            return null;
        }

        //TODO get N users with max measure and work only with them

        for(Long userId : userChoices.keySet())
            if(!userId.equals(account.id))
                for(Long questionId : userChoices.get(userId).keySet())
                    userChoices.get(userId).put(questionId, userChoices.get(userId).get(questionId) * measure.get(userId));

        //questionId -> global measure
        Map<Long,Double> result = new HashMap<>();//Collections.emptyMap();
        for(Long userId : userChoices.keySet())
            if(!userId.equals(account.id))
                for(Long questionId : userChoices.get(userId).keySet()){
                    if(result.containsKey(questionId))
                        result.put(questionId, result.get(questionId)
                                + userChoices.get(userId).get(questionId) / measureSum);
                    else
                        result.put(questionId,userChoices.get(userId).get(questionId)  / measureSum);
                }

        //find max
        Long id = 0L;// = result.keySet().iterator().next();
        double max = 0.0;
        Question question;
        allAnswers = false;
        for(Long questionId : result.keySet()){
            try {
                question = Factory.getInstance().getQuestionDAO().getQuestionById(questionId);
            }catch (SQLException ex) {
                logger.error("Error occurred when getting question: " + ex.getMessage());
                return null;
            }
            if((result.get(questionId) >= max) && (question.level.equals(level)) && (!userAlreadyAnswers(account,questionId,answerRecords)))  {
                max = result.get(questionId);
                id = questionId;
            }
        }
        if(id == 0L){
            allAnswers = true;
            for(Long questionId : result.keySet()){
                try {
                    question = Factory.getInstance().getQuestionDAO().getQuestionById(questionId);
                }catch (SQLException ex) {
                    logger.error("Error occurred when getting question: " + ex.getMessage());
                    return null;
                }
//                System.out.println(questionId + " " + result.get(questionId)+" "+(question.level.equals(level)) +" "+ (!userAlreadyAnswers(user,questionId,answerRecords)));
                if((result.get(questionId) >= max) && (question.level.equals(level)) && (!userAlreadyAnswers(account,questionId,answerRecords))) {
                    max = result.get(questionId);
                    id = questionId;
                }
            }
        }
        System.out.print("id " + id + allAnswers + " " + max);

        //TODO check if id is still 0L
//        System.out.println(measureSum);
        try {
            question = Factory.getInstance().getQuestionDAO().getQuestionById(id);
        } catch (SQLException ex) {
            logger.error("Error occurred when getting question: " + ex.getMessage());
            return null;
        }
        session.close();
        return question;
    }

    private static  Map<Long,Map<Long,Double>> getChoicesRating(List<AnswerRecord> answerRecords){
        //get all question and user Ids
        List<Long> questionIds = session
                .createCriteria(Question.class)
                .setProjection(Projections.property("id"))
                .list();
        List<Long> userIds = session
                .createCriteria(Account.class)
                .setProjection(Projections.property("id"))
                .list();

        //userId->(questionId ->rating)
        //rating:
        //1 if incorrect answer<commit_msg>Algorithm. Filter by topic<commit_after>package algorithm;

import controllers.Factory;
import models.core.*;
import org.hibernate.Session;
import org.hibernate.criterion.Projections;
import play.Logger;
import util.HibernateUtil;

import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Created by ilia on 31.10.14.
 */
public class CollaborativeFiltering {

    private static final Logger.ALogger logger = Logger.of(CollaborativeFiltering.class);

    private static Long level;
    private static Rule rule;
    private static double progress;
    private static double maxProgress = 0.8;
    private static boolean allAnswers = false;
    private static Session session;

    public static Question getNextQuestion(Account account,Long accountLevel, double accountProgress,Topic topic){
        session = HibernateUtil.getSessionFactory().openSession();
        List<AnswerRecord> answerRecords;
        try{
            answerRecords = (ArrayList) Factory.getInstance().getAnswerRecordDAO().getAllAnswerRecords();
        }catch (SQLException ex) {
            logger.error("Error occurred when getting all answerRecords: " + ex.getMessage());
            return null;
        }

//        getCurrentProgress(account,answerRecords);
        level = accountLevel;// account.getAccountLevel();
        if(level == null) level = 1L;
        progress = accountProgress;//account.getLevelProgress();
        if(progress >= maxProgress){
            level++;
            allAnswers = false;
            progress = 0.0;
        }

        //userId->(questionId->rating))
        Map<Long,Map<Long,Double>> userChoices = getChoicesRating(answerRecords);

        //cos measure
        Map<Long,Double> measure = new HashMap<>();
        double measureSum = 0.0;

        double userNorm = 0.0;
        for(Long questionId : userChoices.get(account.id).keySet())
            userNorm += userChoices.get(account.id).get(questionId) * userChoices.get(account.id).get(questionId);
        userNorm = Math.sqrt(userNorm);

        for(Long userId : userChoices.keySet()){
            if(!userId.equals(account.id)){
                double temp = 0.0;  //scalar mult
                double norm = 0.0;
                for(Long questionId : userChoices.get(userId).keySet()){
                    temp += userChoices.get(userId).get(questionId) * userChoices.get(account.id).get(questionId);
                    norm += userChoices.get(userId).get(questionId) * userChoices.get(userId).get(questionId);
                }
                norm = Math.sqrt(norm);
                if(norm != 0.0){
                    measure.put(userId,temp / (norm * userNorm));
                    measureSum += temp / (norm * userNorm);
                }
                else measure.put(userId,0.0);
            }
        }

        if(measureSum == 0.0){
            logger.error("MeasureSum is ZERO");
            return null;
        }

        //TODO get N users with max measure and work only with them

        for(Long userId : userChoices.keySet())
            if(!userId.equals(account.id))
                for(Long questionId : userChoices.get(userId).keySet())
                    userChoices.get(userId).put(questionId, userChoices.get(userId).get(questionId) * measure.get(userId));

        //questionId -> global measure
        Map<Long,Double> result = new HashMap<>();//Collections.emptyMap();
        for(Long userId : userChoices.keySet())
            if(!userId.equals(account.id))
                for(Long questionId : userChoices.get(userId).keySet()){
                    if(result.containsKey(questionId))
                        result.put(questionId, result.get(questionId)
                                + userChoices.get(userId).get(questionId) / measureSum);
                    else
                        result.put(questionId,userChoices.get(userId).get(questionId)  / measureSum);
                }

        //find max
        Long id = 0L;// = result.keySet().iterator().next();
        double max = 0.0;
        Question question;
        allAnswers = false;
        for(Long questionId : result.keySet()){
            try {
                question = Factory.getInstance().getQuestionDAO().getQuestionById(questionId);
            }catch (SQLException ex) {
                logger.error("Error occurred when getting question: " + ex.getMessage());
                return null;
            }
            if((topic != null && question.rule.topic.id.equals(topic.id)) || topic == null)
                if((result.get(questionId) >= max)  && (question.level.equals(level)) && (!userAlreadyAnswers(account,questionId,answerRecords)))  {
                    max = result.get(questionId);
                    id = questionId;
                }
        }
        if(id == 0L){
            allAnswers = true;
            for(Long questionId : result.keySet()){
                try {
                    question = Factory.getInstance().getQuestionDAO().getQuestionById(questionId);
                }catch (SQLException ex) {
                    logger.error("Error occurred when getting question: " + ex.getMessage());
                    return null;
                }
//                System.out.println(questionId + " " + result.get(questionId)+" "+(question.level.equals(level)) +" "+ (!userAlreadyAnswers(user,questionId,answerRecords)));
                if((topic != null && question.rule.topic.id.equals(topic.id)) || topic == null)
                    if((result.get(questionId) >= max) && (question.level.equals(level)) && (!userAlreadyAnswers(account,questionId,answerRecords))) {
                        max = result.get(questionId);
                        id = questionId;
                    }
            }
        }
        System.out.print("id " + id + allAnswers + " " + max);

        if(id == 0L) return null;
//        System.out.println(measureSum);
        try {
            question = Factory.getInstance().getQuestionDAO().getQuestionById(id);
        } catch (SQLException ex) {
            logger.error("Error occurred when getting question: " + ex.getMessage());
            return null;
        }
        session.close();
        return question;
    }

    private static  Map<Long,Map<Long,Double>> getChoicesRating(List<AnswerRecord> answerRecords){
        //get all question and user Ids
        List<Long> questionIds = session
                .createCriteria(Question.class)
                .setProjection(Projections.property("id"))
                .list();
        List<Long> userIds = session
                .createCriteria(Account.class)
                .setProjection(Projections.property("id"))
                .list();

        //userId->(questionId ->rating)
        //rating:
        //1 if incorrect answer
{'fraction_non_alphanumeric': 0.11995565410199556, 'fraction_numerical': 0.004582409460458241, 'mean_word_length': 2.01963847355501, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7474508', 'n_tokens_mistral': 3692, 'n_tokens_neox': 3499, 'n_words': 712}