<commit_before>package knub.master_thesis.probabilistic

import cc.mallet.util.Maths

object Divergence {

    /**
      * The smaller, the more similar. Zero means exact
      */
    def jensenShannonDivergence(p: Array[Double], q: Array[Double]): Double = {
        Maths.jensenShannonDivergence(p, q)
    }

    /**
      * The smaller, the more similar. Zero means exact
      */
    def maxDistance(p: Array[Double], q: Array[Double]): Double = {
        p.indices.map { i => Math.abs(p(i) - q(i)) }.max
    }
    /**
      * The smaller, the more similar. Zero means exact
      */
    def sumDistance(p: Array[Double], q: Array[Double]): Double = {
        0.5 * p.indices.map { i => Math.abs(p(i) - q(i)) }.sum
    }

    /**
      * The smaller, the more similar. Zero means exact
      */
    def bhattacharyyaDistance(p: Array[Double], q: Array[Double]): Double = {
        val BC = p.indices.map { i => Math.sqrt(p(i) * q(i)) }.sum
        - Math.log(BC)
    }

    val sqrt2Rez = 1.0 / Math.sqrt(2)
    /**
      * The smaller, the more similar. Zero means exact
      */
    def hellingerDistance(p: Array[Double], q: Array[Double]): Double = {
        val innerSum = p.indices.map { i =>
            val sqrtDiff = Math.sqrt(p(i)) - Math.sqrt(q(i))
            sqrtDiff * sqrtDiff
        }.sum
        sqrt2Rez * Math.sqrt(innerSum)
    }


//    def kullbackLeibler(p: Array[Double], q: Array[Double]): Double = {
//        var sum = 0.0
//        for (i <- p.indices) {
//            if (p(i) != 0.0 && q(i) != 0.0)
//                sum += p(i) * Math.log(p(i) / q(i))
//        }
//        sum
//    }

}<commit_msg>More efficient implementations for divergences<commit_after>package knub.master_thesis.probabilistic

import cc.mallet.util.Maths

object Divergence {

    /**
      * The smaller, the more similar. Zero means exact
      */
    def jensenShannonDivergence(p: Array[Double], q: Array[Double]): Double = {
        Maths.jensenShannonDivergence(p, q)
    }

    /**
      * The smaller, the more similar. Zero means exact
      */
    def maxDistance(p: Array[Double], q: Array[Double]): Double = {
        var max = 0.0
        var i = 0
        while (i < p.length) {
            val diff = Math.abs(p(i) - q(i))
            if (diff > max)
                max = diff
            i += 1
        }
        max
    }
    /**
      * The smaller, the more similar. Zero means exact
      */
    def sumDistance(p: Array[Double], q: Array[Double]): Double = {
        var sum = 0.0
        var i = 0
        while (i < p.length) {
            sum += Math.abs(p(i) - q(i))
            i += 1
        }
        0.5 * sum
    }

    /**
      * The smaller, the more similar. Zero means exact
      */
    def bhattacharyyaDistance(p: Array[Double], q: Array[Double]): Double = {
        var sum = 0.0
        var i = 0
        while (i < p.length) {
            sum += Math.sqrt(p(i) * q(i))
            i += 1
        }
        - Math.log(sum)
    }

    val sqrt2Rez = 1.0 / Math.sqrt(2)
    /**
      * The smaller, the more similar. Zero means exact
      */
    def hellingerDistance(p: Array[Double], q: Array[Double]): Double = {
        var sum = 0.0
        var i = 0
        while (i < p.length) {
            val sqrtDiff = Math.sqrt(p(i)) - Math.sqrt(q(i))
            sum += sqrtDiff * sqrtDiff
            i += 1
        }
        sqrt2Rez * Math.sqrt(sum)
    }

}
{'fraction_non_alphanumeric': 0.14491053094749193, 'fraction_numerical': 0.010560281607509533, 'mean_word_length': 1.6829268292682926, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2372786', 'n_tokens_mistral': 1186, 'n_tokens_neox': 1134, 'n_words': 294}