<commit_before>"""Yan et al. (2010) EM crowd passive learning algorithm.

<NAME>
The Australian National University
2016
"""

import itertools
import logging

import numpy
import scipy.optimize
import scipy.special
import sklearn.linear_model


def logistic_regression(a, b, x):
    """Logistic regression classifier.

    a: Weights α. (n_dim,) NumPy array
    b: Bias β. float
    x: Data point x_i. (n_dim,) NumPy array
    -> float in [0, 1]
    """
    res = scipy.special.expit(numpy.dot(x, a.T) + b)
    return res


def annotator_model(w, g, x, y, z):
    """Yan et al. (2010) Bernoulli annotator model.

    w: Annotator weights w_t. (n_dim,) NumPy array
    g: Annotator bias γ_t. (n_dim,) NumPy array
    x: Data point x_i. (n_dim,) NumPy array
    y: Label y_i^(t). int
    z: "True" label z_i. int
    -> float in [0, 1]
    """
    eta = logistic_regression(w, g, x) + 0.0001
    label_difference = numpy.abs(y - z)
    return (numpy.power(1 - eta, label_difference.T)
            * numpy.power(eta, 1 - label_difference.T)).T


def unpack(params, n_dim, n_annotators):
    """Unpacks an array of parameters in to a, b, w, and g."""
    a = params[:n_dim]
    b = params[n_dim]
    w = params[n_dim+1:n_dim+1+n_annotators*n_dim].reshape(
            (n_annotators, n_dim))
    g = params[n_dim+1+n_annotators*n_dim:]
    return a, b, w, g


def pack(a, b, w, g):
    """Packs a, b, w, and g into an array of parameters."""
    return numpy.hstack([a, [b], w.ravel(), g])


def Q(params, n_dim, n_annotators, n_samples, posteriors, posteriors_0, x, y):
    """Maximisation step minimisation target."""
    a, b, w, g = unpack(params, n_dim, n_annotators)

    expectation = (
            posteriors.dot((numpy.log(annotator_model(w, g, x, y, 1)) +
                            numpy.log(logistic_regression(a, b, x))).T) +
            posteriors_0.dot((numpy.log(annotator_model(w, g, x, y, 0)) +
                              numpy.log(1 - logistic_regression(a, b, x))).T)
    ).sum()

    # Also need the gradients.
    dp = posteriors - posteriors_0
    # logit_i = scipy.special.expit(x.dot(a) + b)
    dQ_db = n_annotators * (
            posteriors.dot(logistic_regression(-a, -b, x)) +
            posteriors_0.dot(logistic_regression(-a, -b, x) - 1))
    dQ_da = n_annotators * (
            numpy.dot(posteriors * logistic_regression(-a, -b, x) +
                      posteriors_0 * (logistic_regression(-a, -b, x) - 1),
                      x))

    # logit_t = scipy.special.expit(numpy.dot(w, x.T) + g.reshape((-1, 1)))
    dQ_dw = numpy.zeros(w.shape)
    # Inefficient, but unrolled for clarity.
    for t in range(n_annotators):
        dQ_dw[t] += sum(x[i] * posteriors[i] *
                                (logistic_regression(-w[t], -g[t], x[i]) -
                                 abs(y[t, i] - 1)) +
                        x[i] * posteriors_0[i] *
                                (logistic_regression(-w[t], -g[t], x[i]) -<commit_msg>Probably fixed numeric instability. Resolves #151.<commit_after>"""Yan et al. (2010) EM crowd passive learning algorithm.

<NAME>
The Australian National University
2016
"""

import itertools
import logging

import numpy
import scipy.optimize
import scipy.special
import sklearn.linear_model


EPS = 1E-8


def logistic_regression(a, b, x):
    """Logistic regression classifier.

    a: Weights α. (n_dim,) NumPy array
    b: Bias β. float
    x: Data point x_i. (n_dim,) NumPy array
    -> float in [0, 1]
    """
    res = scipy.special.expit(numpy.dot(x, a.T) + b)
    return res


def annotator_model(w, g, x, y, z):
    """Yan et al. (2010) Bernoulli annotator model.

    w: Annotator weights w_t. (n_dim,) NumPy array
    g: Annotator bias γ_t. (n_dim,) NumPy array
    x: Data point x_i. (n_dim,) NumPy array
    y: Label y_i^(t). int
    z: "True" label z_i. int
    -> float in [0, 1]
    """
    eta = logistic_regression(w, g, x)
    label_difference = numpy.abs(y - z)
    return (numpy.power(1 - eta, label_difference.T)
            * numpy.power(eta, 1 - label_difference.T)).T


def unpack(params, n_dim, n_annotators):
    """Unpacks an array of parameters in to a, b, w, and g."""
    a = params[:n_dim]
    b = params[n_dim]
    w = params[n_dim+1:n_dim+1+n_annotators*n_dim].reshape(
            (n_annotators, n_dim))
    g = params[n_dim+1+n_annotators*n_dim:]
    return a, b, w, g


def pack(a, b, w, g):
    """Packs a, b, w, and g into an array of parameters."""
    return numpy.hstack([a, [b], w.ravel(), g])


def Q(params, n_dim, n_annotators, n_samples, posteriors, posteriors_0, x, y):
    """Maximisation step minimisation target."""
    a, b, w, g = unpack(params, n_dim, n_annotators)

    expectation = (
            posteriors.dot((numpy.log(annotator_model(w, g, x, y, 1) + EPS) +
                            numpy.log(logistic_regression(a, b, x) + EPS)).T) +
            posteriors_0.dot((numpy.log(annotator_model(w, g, x, y, 0) + EPS) +
                              numpy.log(1 - logistic_regression(a, b, x) + EPS)
                             ).T)
    ).sum()

    # Also need the gradients.
    dp = posteriors - posteriors_0
    # logit_i = scipy.special.expit(x.dot(a) + b)
    dQ_db = n_annotators * (
            posteriors.dot(logistic_regression(-a, -b, x)) +
            posteriors_0.dot(logistic_regression(-a, -b, x) - 1))
    dQ_da = n_annotators * (
            numpy.dot(posteriors * logistic_regression(-a, -b, x) +
                      posteriors_0 * (logistic_regression(-a, -b, x) - 1),
                      x))

    # logit_t = scipy.special.expit(numpy.dot(w, x.T) + g.reshape((-1, 1)))
    dQ_dw = numpy.zeros(w.shape)
    # Inefficient, but unrolled for clarity.
    for t in range(n_annotators):
        dQ_dw[t] += sum(x[i] * posteriors[i] *
                                (logistic_regression(-w[t], -g[t], x[i]) -
                                 abs(y[t, i] - 1)) +
                        x[i] * posteriors_0[i] *
                                (logistic_regression(-w[t], -g[t], x[i]) -
{'fraction_non_alphanumeric': 0.14470412729985083, 'fraction_numerical': 0.013260401127134096, 'mean_word_length': 2.2423428264373992, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5703685', 'n_tokens_mistral': 2351, 'n_tokens_neox': 2231, 'n_words': 561}