<commit_before>"""
.. py:module:: kappa
   :synopsis: Inter-rater agreement functions.

.. moduleauthor:: <NAME> <<EMAIL>>
"""

import logging

def Fleiss(frequency_matrix:list(list([int]))) -> float:
    """
    Compute Fleiss' Kappa for two or more raters.

    .. seealso::

        `WikiPedia on Fleiss' Kappa <http://en.wikipedia.org/wiki/Fleiss_kappa>`_

    Each row ``s`` in the *frequency matrix* ``M`` represents a subject the
    raters voted on. Each subject row then has a list exactly as long as
    there are rating categories ``c``, holding the number of raters that
    voted for the respective category:

        M\ :sub:`s,c`

    Each rater ``r`` must have voted exactly once on each subject. So these
    sum of each row ``s`` in the matrix must be the same and equal to the
    total number of raters ``R``:

        ∀s: ∑M\ :sub:`s` = R

    And the value each column ``c`` must be in the range [0..R]:

        ∀s,c: 0 ≤ M\ :sub:`s,c` ≤ R

    :param frequency_matrix: A matrix of subjects × categories.
    :raises AssertionError, IndexError: If any row in the *frequency matrix*
                                        has a different `sum` or `len` than
                                        the first.
    """
    M = frequency_matrix
    R = FindAndCheckRaters(M)
    S = len(M)
    C = len(M[0])

    # Compute proportion of votes per category: p_c
    p = [0.0] * C
    SR = S * R

    for c in range(C):
        for s in range(S):
            p[c] += M[s][c]
        
        p[c] /= SR

    # Compute rater agreement per subject: P_s
    P = [0.0] * S
    R2ub = R * (R - 1)

    for s in range(S):
        for c in range(C):
            P[s] += M[s][c] * M[s][c]
        
        P[s] = (P[s] - R) / R2ub

    # Compute the mean [agreement] of the P_s's
    mean_agreement = sum(P) / S

    # Compute the MSE of the proportion of votes
    var_votes = sum(p_c * p_c for p_c in p)

    # Return Fleiss' Kappa
    return (mean_agreement - var_votes) / (1 - var_votes)

def CreateRatingMatrix(ratings:list(dict([(str, str)]))) -> list(list([int])):
    """
    :param ratings: A list of {subject: vote} dictionaries; one per rater.
    :return list: A rating frequency matrix as required for the Kappa
                  functions.
    """
    # Determine subjects
    assert len(ratings) > 1, "at least two raters' results required"
    subjects = set(ratings[0].keys())
    assert all(set(r.keys()) == subjects for r in ratings), \
        "non-equal subject IDs in rating files"
    subjects = sorted(subjects)
    logging.debug("Subjects: %s", subjects)
    logging.info("Total: %s subjects", len(subjects))

    # Determine categories
    categories = set()
    for r in ratings: categories.update(r.values())
    categories = sorted(categories)
    logging.info("Categories: %s", categories)

    # Create and return rating frequency matrix
    #noinspection PyUnusedLocal
    M = [[0] * len(categories) for i in range(len(subjects))]

    for r in ratings:
        for i, s in enumerate(subjects):
            j = categories.index(r[s])
            M[i][j] += 1

    return M

def FindAndCheckRaters(M:list(list([int])))->int:
    """
    Find the number of raters ``R`` and assert that the sum of each row has
    that same number.

    :raises AssertionError: If any row contains a different sum of raters or
                            a different number of categories (ie. length).<commit_msg>had to change the function annotations, as the unittest FW was having problems with them<commit_after>"""
.. py:module:: kappa
   :synopsis: Inter-rater agreement functions.

.. moduleauthor:: <NAME> <florian.<EMAIL>>
"""

import logging

def Fleiss(frequency_matrix):
    """
    Compute Fleiss' Kappa (a `float` value) for two or more raters.

    .. seealso::

        `WikiPedia on Fleiss' Kappa <http://en.wikipedia.org/wiki/Fleiss_kappa>`_

    Each row ``s`` in the *frequency matrix* ``M`` represents a subject the
    raters voted on. Each subject row then has a list exactly as long as
    there are rating categories ``c``, holding the number of raters that
    voted for the respective category:

        M\ :sub:`s,c`

    Each rater ``r`` must have voted exactly once on each subject. So these
    sum of each row ``s`` in the matrix must be the same and equal to the
    total number of raters ``R``:

        ∀s: ∑M\ :sub:`s` = R

    And the value each column ``c`` must be in the range [0..R]:

        ∀s,c: 0 ≤ M\ :sub:`s,c` ≤ R

    :param frequency_matrix: A matrix of subjects × categories.
    :raises AssertionError, IndexError: If any row in the *frequency matrix*
                                        has a different `sum` or `len` than
                                        the first.
    """
    M = frequency_matrix
    R = FindAndCheckRaters(M)
    S = len(M)
    C = len(M[0])

    # Compute proportion of votes per category: p_c
    p = [0.0] * C
    SR = S * R

    for c in range(C):
        for s in range(S):
            p[c] += M[s][c]
        
        p[c] /= SR

    # Compute rater agreement per subject: P_s
    P = [0.0] * S
    R2ub = R * (R - 1)

    for s in range(S):
        for c in range(C):
            P[s] += M[s][c] * M[s][c]
        
        P[s] = (P[s] - R) / R2ub

    # Compute the mean [agreement] of the P_s's
    mean_agreement = sum(P) / S

    # Compute the MSE of the proportion of votes
    var_votes = sum(p_c * p_c for p_c in p)

    # Return Fleiss' Kappa
    return (mean_agreement - var_votes) / (1 - var_votes)

def CreateRatingMatrix(ratings:[{str: str}]) -> [[int]]:
    """
    :param ratings: A list of {subject: vote} dictionaries; one per rater.
    :return list: A rating frequency matrix as required for the Kappa
                  functions.
    """
    # Determine subjects
    assert len(ratings) > 1, "at least two raters' results required"
    subjects = set(ratings[0].keys())
    assert all(set(r.keys()) == subjects for r in ratings), \
        "non-equal subject IDs in rating files"
    subjects = sorted(subjects)
    logging.debug("Subjects: %s", subjects)
    logging.info("Total: %s subjects", len(subjects))

    # Determine categories
    categories = set()
    for r in ratings: categories.update(r.values())
    categories = sorted(categories)
    logging.info("Categories: %s", categories)

    # Create and return rating frequency matrix
    #noinspection PyUnusedLocal
    M = [[0] * len(categories) for i in range(len(subjects))]

    for r in ratings:
        for i, s in enumerate(subjects):
            j = categories.index(r[s])
            M[i][j] += 1

    return M

def FindAndCheckRaters(M:[[int]]) -> int:
    """
    Find the number of raters ``R`` and assert that the sum of each row has
    that same number.

    :raises AssertionError: If any row contains a different sum of raters or
                            a different number of categories (ie. length).
{'fraction_non_alphanumeric': 0.1175534991324465, 'fraction_numerical': 0.004337767495662233, 'mean_word_length': 2.315915627996165, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3669012', 'n_tokens_mistral': 2230, 'n_tokens_neox': 2140, 'n_words': 808}