<commit_before>/*
 *
 *  * Copyright 2015 Skymind,Inc.
 *  *
 *  *    Licensed under the Apache License, Version 2.0 (the "License");
 *  *    you may not use this file except in compliance with the License.
 *  *    You may obtain a copy of the License at
 *  *
 *  *        http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *    Unless required by applicable law or agreed to in writing, software
 *  *    distributed under the License is distributed on an "AS IS" BASIS,
 *  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *    See the License for the specific language governing permissions and
 *  *    limitations under the License.
 *
 */

package org.deeplearning4j.eval;

import java.text.DecimalFormat;
import java.util.Set;

import org.deeplearning4j.berkeley.Counter;
import org.nd4j.linalg.api.ndarray.INDArray;

/**
 * Evaluation metrics: precision, recall, f1
 * @author <NAME>
 *
 */
public class Evaluation {

	private Counter<Integer> truePositives = new Counter<>();
	private Counter<Integer> falsePositives = new Counter<>();
	private Counter<Integer> trueNegative = new Counter<>();
	private Counter<Integer> falseNegatives = new Counter<>();
	private ConfusionMatrix<Integer> confusion = new ConfusionMatrix<>();

	/**
	 * Collects statistics on the real outcomes vs the 
	 * guesses. This is for logistic outcome matrices such that the 
	 * 
	 * Note that an IllegalArgumentException is thrown if the two passed in
	 * matrices aren't the same length.
	 * @param realOutcomes the real outcomes (usually binary)
	 * @param guesses the guesses (usually a probability vector)
	 */
	public void eval(INDArray realOutcomes,INDArray guesses) {
		if(realOutcomes.length() != guesses.length())
			throw new IllegalArgumentException("Unable to evaluate. Outcome matrices not same length");
		for(int i = 0; i < realOutcomes.rows(); i++) {
			INDArray currRow = realOutcomes.getRow(i);
			INDArray guessRow = guesses.getRow(i);

			int currMax;
            {
                double max = currRow.getDouble(0);
                currMax = 0;
                for (int col = 1; col < currRow.columns(); col++) {
                    if (currRow.getDouble(col) > max) {
                        max = currRow.getDouble(col);
                        currMax = col;
                    }
                }
            }
            int guessMax;
            {
                double max = guessRow.getDouble(0);
                guessMax = 0;
                for (int col = 1; col < guessRow.columns(); col++) {
                    if (guessRow.getDouble(col) > max) {
                        max = guessRow.getDouble(col);
                        guessMax = col;
                    }
                }
            }

			addToConfusion(currMax,guessMax);

			if(currMax == guessMax) {
				incrementTruePositives(guessMax);
				for(Integer clazz : confusion.getClasses()) {
					if(clazz != guessMax)
						trueNegative.incrementCount(clazz, 1.0);
				}
			}
			else {
				incrementFalseNegatives(currMax);
				incrementFalsePositives(guessMax);
			}
			
			
			
		}
	}



	public String stats() {
		StringBuilder builder = new StringBuilder()
		.append("\n");
		Set<Integer> classes = confusion.getClasses();
		for(Integer clazz : classes) {
			for(Integer clazz2 : classes) {
				int count = confusion.getCount(clazz, clazz2);
				if(count != 0)
					builder.append("\nActual Class " + clazz + " was predicted with Predicted " + clazz2 + " with count " + count  + " times\n");
			}
		}
		DecimalFormat df = new DecimalFormat("#.####");
		builder.append("\n==========================Scores========================================");
		builder.append("\n Accuracy:  " + df.format(accuracy()));
		builder.append("\n Precision: " + df.format(precision()));
		builder.append("\n Recall:    " + df.format(recall()));
		builder.append("\n F1 Score:  " + df.format(f1()));
		builder.append("\n===========================================================================");
		return builder.toString();
	}

	/**
	 * Adds to the confusion matrix
	 * @param real the actual guess
	 * @param guess the system guess
	 */<commit_msg>mend<commit_after>/*
 *
 *  * Copyright 2015 Skymind,Inc.
 *  *
 *  *    Licensed under the Apache License, Version 2.0 (the "License");
 *  *    you may not use this file except in compliance with the License.
 *  *    You may obtain a copy of the License at
 *  *
 *  *        http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *    Unless required by applicable law or agreed to in writing, software
 *  *    distributed under the License is distributed on an "AS IS" BASIS,
 *  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *    See the License for the specific language governing permissions and
 *  *    limitations under the License.
 *
 */

package org.deeplearning4j.eval;

import java.text.DecimalFormat;
import java.util.Set;

import org.deeplearning4j.berkeley.Counter;
import org.nd4j.linalg.api.ndarray.INDArray;

/**
 * Evaluation metrics: precision, recall, f1
 * @author <NAME>
 *
 */
public class Evaluation {

	private Counter<Integer> truePositives = new Counter<>();
	private Counter<Integer> falsePositives = new Counter<>();
	private Counter<Integer> trueNegative = new Counter<>();
	private Counter<Integer> falseNegatives = new Counter<>();
	private ConfusionMatrix<Integer> confusion = new ConfusionMatrix<>();

	/**
	 * Collects statistics on the real outcomes vs the 
	 * guesses. This is for logistic outcome matrices such that the 
	 * 
	 * Note that an IllegalArgumentException is thrown if the two passed in
	 * matrices aren't the same length.
	 * @param realOutcomes the real outcomes (usually binary)
	 * @param guesses the guesses (usually a probability vector)
	 */
	public void eval(INDArray realOutcomes,INDArray guesses) {
		if(realOutcomes.length() != guesses.length())
			throw new IllegalArgumentException("Unable to evaluate. Outcome matrices not same length");
		for(int i = 0; i < realOutcomes.rows(); i++) {
			INDArray currRow = realOutcomes.getRow(i);
			INDArray guessRow = guesses.getRow(i);

			int currMax;
            {
                double max = currRow.getDouble(0);
                currMax = 0;
                for (int col = 1; col < currRow.columns(); col++) {
                    if (currRow.getDouble(col) > max) {
                        max = currRow.getDouble(col);
                        currMax = col;
                    }
                }
            }
            int guessMax;
            {
                double max = guessRow.getDouble(0);
                guessMax = 0;
                for (int col = 1; col < guessRow.columns(); col++) {
                    if (guessRow.getDouble(col) > max) {
                        max = guessRow.getDouble(col);
                        guessMax = col;
                    }
                }
            }

			addToConfusion(currMax,guessMax);

			if(currMax == guessMax) {
				incrementTruePositives(guessMax);
				for(Integer clazz : confusion.getClasses()) {
					if(clazz != guessMax)
						trueNegative.incrementCount(clazz, 1.0);
				}
			}
			else {
				incrementFalseNegatives(currMax);
				incrementFalsePositives(guessMax);
			}
			
			
			
		}
	}



	public String stats() {
		StringBuilder builder = new StringBuilder()
		.append("\n");
		Set<Integer> classes = confusion.getClasses();
		for(Integer clazz : classes) {
			for(Integer clazz2 : classes) {
				int count = confusion.getCount(clazz, clazz2);
				if(count != 0)
					builder.append("\nActual Class " + clazz + " was predicted with Predicted " + clazz2 + " with count " + count  + " times\n");
			}
		}
		DecimalFormat df = new DecimalFormat("#.####");
		builder.append("\n==========================Scores========================================");
		builder.append("\n Accuracy:  " + df.format(accuracy()));
		builder.append("\n Precision: " + df.format(precision()));
		builder.append("\n Recall:    " + df.format(recall()));
		builder.append("\n F1 Score:  " + f1());
		builder.append("\n===========================================================================");
		return builder.toString();
	}

	/**
	 * Adds to the confusion matrix
	 * @param real the actual guess
	 * @param guess the system guess
	 */
{'fraction_non_alphanumeric': 0.14738491202699447, 'fraction_numerical': 0.006507592190889371, 'mean_word_length': 2.617698343504795, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7662051', 'n_tokens_mistral': 2598, 'n_tokens_neox': 2287, 'n_words': 664}