<commit_before>"""
Utility classes and functions related to computing test
theory based evaluations

:author: <NAME> (<EMAIL>)
:author: <NAME> (<EMAIL>)
:author: <NAME> (<EMAIL>)

:date: 05/20/2019
:organization: ETS
"""

import pandas as pd


def compute_variance_of_errors(df,
                               h1_column='sc1',
                               h2_column='sc2',):
    """
    Compute variance of errors in human scores.

    Parameters
    ----------
    df : pandas DataFrame
        Input data frame. Must contain columns `h1_column` and `h2_column`.
    h1_column : str, optional
        The first human score column name.
        Defaults to 'sc1'.
    h2_column : str, optional
        The second human score column name.
        Defaults to 'sc2'.

    Returns
    -------
    variance_errors_human: float
         Variance of errors in human scores

    Raises
    ------
    ValueError
         If any of the values in `h2_column` or `h1_column` are NaN.
    """

    # check that all responses are double scored
    if df[h2_column].isnull().any() or df[h1_column].isnull().any():
        raise ValueError("Variance of errors should only be computed on double-scored responses.")

    N = len(df)

    # estimate variance of errors in human scores for double-scored responses
    # as 1/2 of average squared difference between the two scores
    variance_errors_human = 1 / (2 * N) * ((df[h1_column] - df[h2_column])**2).sum()

    return variance_errors_human


def compute_true_score_var_subset_double_scored(single_human_scores,
                                                double_human_scores,
                                                variance_errors_human):
    """
    Compute variance of true scores
    in a situation where only some
    of the responses are double scored.

    Parameters
    ----------
    single_human_scores : pandas Series
        Human scores for single-scored responses
    double_human_scores : pandas Series
       Human scores for double-scored responses
    variance_errors_human : float
        Estimated variance of errors in human scores

    Returns
    -------
    variance_true_scores : float
        Variance of true scores
    """

    # N of examinees with 1 human score.
    n_1 = len(single_human_scores)

    # N of examinees with 2 human scores.
    n_2 = len(double_human_scores)

    N = n_1 + n_2

    # compute squared deviation of sc_bar score for each responses
    # from mean sc_bar score.
    # This is then multiplied by a coefficient corresponding to the number
    # of human scores (1 for single scores responses, 2 for double-scored responses)
    # so that the responses with higher number of human scores are
    # weighed higher

    sc_bar_mean = pd.concat([single_human_scores, double_human_scores]).mean()

    squared_dist_single = (single_human_scores - sc_bar_mean)**2
    squared_dist_double = 2 * (double_human_scores - sc_bar_mean)**2

    # concatenate both data frames
    squared_dist = pd.concat([squared_dist_single, squared_dist_double], sort=True)

    # third, compute variance of true scores.

    # the numerator corresponds to the weighed sum of squares adjusted for
    # the estimated variance or errors in human scores
    numerator = (squared_dist.sum() - (N - 1) * variance_errors_human)

    # the denominator is construed to ensure correct treatment of 
    # all cases regardless of what percentage responses is double-scored
    denominator = (N - 1) + (n_2 * (n_1 + 2 * n_2 - 2)) / (n_1 + 2 * n_2)

    variance_true_scores = numerator / denominator

    return variance_true_scores


def compute_mse_subset_double_scored(single_human_scores,
                                     double_human_scores,
                                     single_system_scores,
                                     double_system_scores,
                                     variance_errors_human):
    """
    Compute MSE when predicting true score from system scores
    in a situation where only some
    of the responses are double scored.

    Parameters
    ----------
    single_human_scores : pandas Series
        Human scores for single-scored responses
    double_human_scores : pandas Series
       Human scores for double-scored responses
    single_system_scores : pandas Series
        System scores for single_scored responses
    double_human_scores : pandas Series
        System scores for double_scored responses
    variance_errors_human : float
        Variance of errors in human scores

    Returns
    -------
    mse : float
       Mean squared error
    """

    # N of examinees with 1 human score.
    n_1 = len(single_human_scores)

    # N of examinees with 2 human scores.
    n_2 = len(double_human_scores)

    N = n_1 + n_2

    # compute squared error c_i*(sc_bar_i - system_i)**2.
    # Note that for double-scored responses c = 2.
    se_single = (single_human_scores - single_system_scores)**2
    se_double = 2 * (double_human_scores - double_system_scores)**2

    # concatenate both sets
    se = pd.concat([se_single, se_double], sort=True)

    # Compute mean squared error when predicting true score
    mse = (se.sum() - N * variance_errors_human) / (n_1 + 2 * n_2)

    return mse


def compute_true_score_var_all_double_scored(human_scores,
                                             variance_errors_human):
    """
    Compute variance of true scores
    in a situation where all
    responses are double scored.

    Parameters
    ----------
    human_scores : pandas Series
        Human scores used to compute the variance
    variance_errors_human : float
        Variance of errors in human scores

    Returns
    -------
    variance_true_scores : float
        Variance of true scores
    """
    N = len(human_scores)

    # compute sum of squared deviations of observed human scores from
    # mean score
    numerator = ((human_scores - human_scores.mean())**2).sum() 

    denominator = (N - 1)

    # compute variance of true scores as variance of observed
    # scores adjusted for estimated variance of errors in human scores
    variance_true_scores = numerator/denominator  - variance_errors_human/ 2

    return variance_true_scores


def compute_mse_all_double_scored(human_scores,
                                  system_scores,
                                  variance_errors_human):
    """
    Compute MSE when predicting true score from system scores
    in a situation where all
    of the responses are double scored.

    Parameters
    ----------
    human_scores : pandas Series
        Human scores
    system_scores : pandas Series
        System scores
    variance_errors_human : float
        Variance of errors in human scores

    Returns
    -------
    mse : float
        Mean squared error<commit_msg>Few minor fixes and typos<commit_after>"""
Utility classes and functions related to computing test
theory based evaluations

:author: <NAME> (<EMAIL>)
:author: <NAME> (<EMAIL>)
:author: <NAME> (<EMAIL>)

:date: 05/20/2019
:organization: ETS
"""

import pandas as pd


def compute_variance_of_errors(df,
                               h1_column='sc1',
                               h2_column='sc2',):
    """
    Compute variance of errors in human scores.

    Parameters
    ----------
    df : pandas DataFrame
        Input data frame. Must contain columns `h1_column` and `h2_column`.
    h1_column : str, optional
        The first human score column name.
        Defaults to 'sc1'.
    h2_column : str, optional
        The second human score column name.
        Defaults to 'sc2'.

    Returns
    -------
    variance_errors_human: float
         Variance of errors in human scores

    Raises
    ------
    ValueError
         If any of the values in `h2_column` or `h1_column` are NaN.
    """

    # check that all responses are double scored
    if df[h2_column].isnull().any() or df[h1_column].isnull().any():
        raise ValueError("Variance of errors should only be computed on double-scored responses.")

    N = len(df)

    # estimate variance of errors in human scores for double-scored responses
    # as 1/2 of average squared difference between the two scores
    variance_errors_human = 1 / (2 * N) * ((df[h1_column] - df[h2_column])**2).sum()

    return variance_errors_human


def compute_true_score_var_subset_double_scored(single_human_scores,
                                                double_human_scores,
                                                variance_errors_human):
    """
    Compute variance of true scores
    in a situation where only some
    of the responses are double scored.

    Parameters
    ----------
    single_human_scores : pandas Series
        Human scores for single-scored responses
    double_human_scores : pandas Series
       Human scores for double-scored responses
    variance_errors_human : float
        Estimated variance of errors in human scores

    Returns
    -------
    variance_true_scores : float
        Variance of true scores
    """

    # N of examinees with 1 human score.
    n_1 = len(single_human_scores)

    # N of examinees with 2 human scores.
    n_2 = len(double_human_scores)

    N = n_1 + n_2

    # compute squared deviation of sc_bar score for each responses
    # from mean sc_bar score.
    # This is then multiplied by a coefficient corresponding to the number
    # of human scores (1 for single scores responses, 2 for double-scored responses)
    # so that the responses with higher number of human scores are
    # weighed higher

    sc_bar_mean = pd.concat([single_human_scores, double_human_scores]).mean()

    squared_dist_single = (single_human_scores - sc_bar_mean)**2
    squared_dist_double = 2 * (double_human_scores - sc_bar_mean)**2

    # concatenate both data frames
    squared_dist = pd.concat([squared_dist_single, squared_dist_double], sort=True)

    # third, compute variance of true scores.

    # the numerator corresponds to the weighed sum of squares adjusted for
    # the estimated variance or errors in human scores
    numerator = (squared_dist.sum() - (N - 1) * variance_errors_human)

    # the denominator is construed to ensure correct treatment of 
    # all cases regardless of what percentage responses is double-scored
    denominator = (N - 1) + (n_2 * (n_1 + 2 * n_2 - 2)) / (n_1 + 2 * n_2)

    variance_true_scores = numerator / denominator

    return variance_true_scores


def compute_mse_subset_double_scored(single_human_scores,
                                     double_human_scores,
                                     single_system_scores,
                                     double_system_scores,
                                     variance_errors_human):
    """
    Compute MSE when predicting true score from system scores
    in a situation where only some
    of the responses are double scored.

    Parameters
    ----------
    single_human_scores : pandas Series
        Human scores for single-scored responses
    double_human_scores : pandas Series
       Human scores for double-scored responses
    single_system_scores : pandas Series
        System scores for single-scored responses
    double_human_scores : pandas Series
        System scores for double-scored responses
    variance_errors_human : float
        Variance of errors in human scores

    Returns
    -------
    mse : float
       Mean squared error
    """

    # N of examinees with 1 human score.
    n_1 = len(single_human_scores)

    # N of examinees with 2 human scores.
    n_2 = len(double_human_scores)

    N = n_1 + n_2

    # compute squared error c_i*(sc_bar_i - system_i)**2.
    # Note that for double-scored responses c = 2.
    se_single = (single_human_scores - single_system_scores)**2
    se_double = 2 * (double_human_scores - double_system_scores)**2

    # concatenate both sets
    se = pd.concat([se_single, se_double], sort=True)

    # Compute mean squared error when predicting true score
    mse = (se.sum() - N * variance_errors_human) / (n_1 + 2 * n_2)

    return mse


def compute_true_score_var_all_double_scored(human_scores,
                                             variance_errors_human):
    """
    Compute variance of true scores
    in a situation where all
    responses are double scored.

    Parameters
    ----------
    human_scores : pandas Series
        Human scores used to compute the variance
    variance_errors_human : float
        Variance of errors in human scores

    Returns
    -------
    variance_true_scores : float
        Variance of true scores
    """
    N = len(human_scores)

    # compute variance of observed human scores:
    variance_observed_scores = ((human_scores - human_scores.mean())**2).sum() / (N-1)

    # compute variance of true scores as variance of observed
    # scores adjusted for estimated variance of errors in human scores
    variance_true_scores = variance_observed_scores - variance_errors_human / 2

    return variance_true_scores


def compute_mse_all_double_scored(human_scores,
                                  system_scores,
                                  variance_errors_human):
    """
    Compute MSE when predicting true score from system scores
    in a situation where all
    of the responses are double scored.

    Parameters
    ----------
    human_scores : pandas Series
        Human scores
    system_scores : pandas Series
        System scores
    variance_errors_human : float
        Variance of errors in human scores

    Returns
    -------
    mse : float
        Mean squared error
{'fraction_non_alphanumeric': 0.0703136504196731, 'fraction_numerical': 0.00986599911647769, 'mean_word_length': 2.291252725951054, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5179934', 'n_tokens_mistral': 3871, 'n_tokens_neox': 3576, 'n_words': 1259}