<commit_before># Provides some universal statistical utilities and stats comparison tools

from math import sqrt

import numpy as np
import pandas as pd
import scipy.interpolate
import scipy.stats
import warnings
from scipy.special import erfinv

from threeML.io.rich_display import display


def aic(log_like, n_parameters, n_data_points):
    """
    The Aikake information criterion.
    A model comparison tool based of infomormation theory. It assumes that N is large i.e.,
    that the model is approaching the CLT.


    """

    val = -2. * log_like + 2 * n_parameters
    val += 2 * n_parameters * (n_parameters + 1) / (n_data_points - n_parameters - 1)

    return val


def bic(log_like, n_parameters, n_data_points):
    """
    The Bayesian information criterion.
    """
    val = -2. * log_like + n_parameters * np.log(n_data_points)

    return val


def waic(bayesian_trace):
    raise NotImplementedError("Coming soon to a theater near you.")


def effective_number_of_parameters(bayesian_trace):
    raise NotImplementedError("Coming soon to a theater near you.")


def dic(bayesian_trace):
    """
    The Deviance information criteria derived from MCMC traces
    Read more:  dx.doi.org/10.1111/1467-9868.00353


    :param bayesian_trace: an instance of Bayesian Analysis

    :return: deviance information criteria

    """

    mean_deviance = -2. * np.mean(bayesian_trace.log_probability_values)

    mean_of_free_parameters = np.mean(bayesian_trace.raw_samples, axis=0)

    deviance_at_mean = -2. * bayesian_trace.get_posterior(mean_of_free_parameters)

    return 2 * mean_deviance - deviance_at_mean


class ModelComparison(object):
    def __init__(self, *analyses):
        self._analysis_container = analyses

        # First make sure that it is all bayesian or all MLE
        assert (np.unique([a.analysis_type for a in analyses])).shape[
                   0] == 1, "Only all Bayesian or all MLE analyses are allowed. Not a mixture!"


        self._analysis_type = analyses[0].analysis_type

        if self._analysis_type == 'mle':

            self._stat_df = self._compute_mle_statistics()



        elif self._analysis_type == 'bayesian':

            self._stat_df = self._compute_bayes_statistics()<commit_msg>added sqrt sum of squares to stats tool<commit_after># Provides some universal statistical utilities and stats comparison tools

from math import sqrt

import numpy as np
import pandas as pd
import scipy.interpolate
import scipy.stats
import warnings
from scipy.special import erfinv

from threeML.io.rich_display import display


def aic(log_like, n_parameters, n_data_points):
    """
    The Aikake information criterion.
    A model comparison tool based of infomormation theory. It assumes that N is large i.e.,
    that the model is approaching the CLT.


    """

    val = -2. * log_like + 2 * n_parameters
    val += 2 * n_parameters * (n_parameters + 1) / (n_data_points - n_parameters - 1)

    return val


def bic(log_like, n_parameters, n_data_points):
    """
    The Bayesian information criterion.
    """
    val = -2. * log_like + n_parameters * np.log(n_data_points)

    return val


def waic(bayesian_trace):
    raise NotImplementedError("Coming soon to a theater near you.")


def effective_number_of_parameters(bayesian_trace):
    raise NotImplementedError("Coming soon to a theater near you.")


def dic(bayesian_trace):
    """
    The Deviance information criteria derived from MCMC traces
    Read more:  dx.doi.org/10.1111/1467-9868.00353


    :param bayesian_trace: an instance of Bayesian Analysis

    :return: deviance information criteria

    """

    mean_deviance = -2. * np.mean(bayesian_trace.log_probability_values)

    mean_of_free_parameters = np.mean(bayesian_trace.raw_samples, axis=0)

    deviance_at_mean = -2. * bayesian_trace.get_posterior(mean_of_free_parameters)

    return 2 * mean_deviance - deviance_at_mean


def sqrt_sum_of_squares(arg):

    return np.sqrt( np.square(arg).sum() )



class ModelComparison(object):
    def __init__(self, *analyses):
        self._analysis_container = analyses

        # First make sure that it is all bayesian or all MLE
        assert (np.unique([a.analysis_type for a in analyses])).shape[
                   0] == 1, "Only all Bayesian or all MLE analyses are allowed. Not a mixture!"


        self._analysis_type = analyses[0].analysis_type

        if self._analysis_type == 'mle':

            self._stat_df = self._compute_mle_statistics()



        elif self._analysis_type == 'bayesian':

            self._stat_df = self._compute_bayes_statistics()
{'fraction_non_alphanumeric': 0.08154413359358056, 'fraction_numerical': 0.013879852526566904, 'mean_word_length': 3.314312441534144, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7459996', 'n_tokens_mistral': 1519, 'n_tokens_neox': 1363, 'n_words': 417}