<commit_before>import java.util.*;
import java.util.stream.Stream;

public class ExpectationMaximization {
    private final static double TESTED_LAMBDA = 0.01; // check
    private final static double EPSILON_THRESHOLD = 0.00000001;
    private final static double K = 10;
    private final static double EM_THRESHOLD = 1; // check

    private Map<Integer, List<Article>> clusters;
    private DevelopmentSet developmentSet;
    private Topics topics;
    private int numClusters;
    private Map<Article, Double[]> Wti;
    private Map<Article, Double[]> Zti;
    private Map<Article, Double> Mt;
    private Map<String, Double[]> Pik;
    private double clustersProbability[]; //alpha(i)

    public void init(DevelopmentSet developmentSet, int numClusters, Topics topics) {
        this.Wti = new HashMap<>();
        this.Zti = new HashMap<>();
        this.Mt = new HashMap<>();
        this.Pik = new HashMap<>();
        this.developmentSet = developmentSet;
        this.topics = topics;
        this.numClusters = numClusters;
        this.clustersProbability = new double[numClusters];

        initClusters();
        initEM();
        MStep();
    }

    public void run() {
        double perplexity;
        double likelihood = 0;
        List<Double> likelihoods = new ArrayList<Double>();
        List<Double> perplexities = new ArrayList<Double>();
        double lastLikelihood = likelihood - EM_THRESHOLD - 1;
        Integer[][] confusionMatrix = buildConfusionMatrix();

        // if in some round
        // we find that the Likelihood decrease - it means that we have a bug in our implementation or
        // that we are smoothing too aggressively.

        // Run EM algorithm until convergence
        while (likelihood - lastLikelihood > EM_THRESHOLD) {
            EStep();
            MStep();

            // Save likelihoods for future graph plot
            lastLikelihood = likelihood;
            likelihood = calcLikelihood();
            likelihoods.add(likelihood);

            // Save likelihoods for future graph plot
            perplexity = calcLikelihood(likelihood);
            perplexities.add(perplexity);
        }
    }

    private Integer[][] buildConfusionMatrix() {
        Integer[][] confusionMatrix = new Integer[this.numClusters][this.numClusters + 1];

        for (Integer[] row : confusionMatrix) {
            Arrays.fill(row, 0);
        }

        int maxCluster;
        for (Article currentArticle : developmentSet.getArticles()) {
            Double maxWt = Wti.get(currentArticle)[0];
            maxCluster = 0;
            for (int i = 1; i < this.numClusters; i++) {
                Double wti = Wti.get(currentArticle)[i];
                if (wti > maxWt) {
                    maxWt = wti;
                    maxCluster = i;
                }
            }
            currentArticle.setAssignedTopic(topics.getTopics()[maxCluster]);

            // Build the confusion matrix based on the given topics and the max cluster topic
            for (String topic : currentArticle.getTopics()) {
                confusionMatrix[maxCluster][topics.getTopicIndex(topic)] += 1;
                confusionMatrix[maxCluster][this.numClusters] += 1;
            }
        }


        return confusionMatrix;
    }

    private double calcLikelihood(double likelihood) {
        return Math.pow(2, -1.0 / developmentSet.countNumberOfWords() * likelihood);
    }

    private void initClusters() {
        final int[] index = {0};
        clusters = new HashMap<>();

        developmentSet.getArticles().forEach(article -> {
            int key = index[0]++ % numClusters;

            if (!clusters.containsKey(key)) {
                clusters.put(key, new ArrayList<>());<commit_msg>Added perplexities<commit_after>import java.util.*;
import java.util.stream.Stream;

public class ExpectationMaximization {
    private final static double TESTED_LAMBDA = 0.01; // check
    private final static double EPSILON_THRESHOLD = 0.00000001;
    private final static double K = 10;
    private final static double EM_THRESHOLD = 1; // check

    private Map<Integer, List<Article>> clusters;
    private DevelopmentSet developmentSet;
    private Topics topics;
    private int numClusters;
    private Map<Article, Double[]> Wti;
    private Map<Article, Double[]> Zti;
    private Map<Article, Double> Mt;
    private Map<String, Double[]> Pik;
    private double clustersProbability[]; //alpha(i)

    public void init(DevelopmentSet developmentSet, int numClusters, Topics topics) {
        this.Wti = new HashMap<>();
        this.Zti = new HashMap<>();
        this.Mt = new HashMap<>();
        this.Pik = new HashMap<>();
        this.developmentSet = developmentSet;
        this.topics = topics;
        this.numClusters = numClusters;
        this.clustersProbability = new double[numClusters];

        initClusters();
        initEM();
        MStep();
    }

    public void run() {
        double likelihood = 0;
        List<Double> likelihoods = new ArrayList<Double>();
        double perplexity = 0;
        List<Double> perplexities = new ArrayList<Double>();
        double lastLikelihood = likelihood - EM_THRESHOLD - 1;
        // if in some round
        // we find that the Likelihood decrease - it means that we have a bug in our implementation or
        // that we are smoothing too aggressively.

        
        // Run EM algorithm until convergence
        while (likelihood - lastLikelihood > EM_THRESHOLD) {
            EStep();
            MStep();
            
            // Save likelihoods for future graph plot
            lastLikelihood = likelihood;
            likelihood = calcLikelihood();
            likelihoods.add(likelihood);
            
            // Save perplexities for future graph plot
            perplexity = calcPerplexity(likelihood);
            perplexities.add(perplexity);
        }
        
        Integer[][] confusionMatrix = bulidConfusionMatrix();
        double accuracy = calcAccuracy(confusionMatrix);
        System.out.println("Accuracy rate is: " + accuracy);
    }

    private double calcAccuracy(Integer[][] confusionMatrix) {
    	int correctAssignments = 0;
    	for (int i=0; i<this.numClusters; i++) {
    		correctAssignments += confusionMatrix[i][i];
    	}
    	
		return correctAssignments / developmentSet.getArticles().size();
	}

	private Integer[][] bulidConfusionMatrix() {
    	Integer[][] confusionMatrix = new Integer[this.numClusters][this.numClusters+1];
    	
        for (Integer[] row: confusionMatrix)
        {
        	Arrays.fill(row, 0);
        }
    	
		int maxCluster;
		for (Article currentArticle : developmentSet.getArticles()) {
			Double maxWt = Wti.get(currentArticle)[0];
			maxCluster = 0;
			for (int i=1; i<this.numClusters; i++){
				Double wti = Wti.get(currentArticle)[i];
				if (wti > maxWt){
					maxWt = wti;
					maxCluster = i;
				}
			}
			currentArticle.setAssignedTopic(topics.getTopics()[maxCluster]);
			
			// Build the confusion matrix based on the given topics and the max cluster topic
			for (String topic : currentArticle.getTopics()) {
				confusionMatrix[maxCluster][topics.getTopicIndex(topic)] += 1;
				confusionMatrix[maxCluster][this.numClusters] += 1;
			}
		}

    	
		return confusionMatrix;
	}

	private double calcPerplexity(double likelihood) {
    	return Math.pow(2, -1.0/developmentSet.countNumberOfWords() * likelihood);
	}

	private void initClusters() {
        final int[] index = {0};
        clusters = new HashMap<>();

        developmentSet.getArticles().forEach(article -> {
            int key = index[0]++ % numClusters;

            if (!clusters.containsKey(key)) {
                clusters.put(key, new ArrayList<>());
{'fraction_non_alphanumeric': 0.1040402736543178, 'fraction_numerical': 0.007874015748031496, 'mean_word_length': 2.384884228920926, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3930339', 'n_tokens_mistral': 2262, 'n_tokens_neox': 2094, 'n_words': 573}