<commit_before>"""Maximally Informative Representations using CORrelation EXplanation

Main ideas first described in:
<NAME> and <NAME>. "Maximally Informative
Hierarchical Representations of High-Dimensional Data"
AISTATS, 2015. arXiv preprint(arXiv:1410.7404.)

The Bayesian smoothing option is described in:
Pepke and Ver Steeg, Comprehensive discovery of subsample gene expression components
by information explanation: therapeutic implications in cancer. BMC Medical Genomics, 2017.

Code below written by: <NAME> (<EMAIL>)

License: Apache V2
"""

from __future__ import print_function
import sys
import numpy as np  # Tested with 1.8.0
from os import makedirs
from os import path
from numpy import ma
from scipy.misc import logsumexp  # Tested with 0.13.0
from multiprocessing import Pool


def unwrap_f(arg):
    """Multiprocessing pool.map requires a top-level function."""
    return Corex.calculate_p_xi_given_y(*arg)

def logsumexp2(z):
    """Multiprocessing pool.map requires a top-level function."""
    return logsumexp(z, axis=2)


class Corex(object):
    """
    Correlation Explanation

    A method to learn a hierarchy of successively more abstract
    representations of complex data that are maximally
    informative about the data. This method is unsupervised,
    requires no assumptions about the data-generating model,
    and scales linearly with the number of variables.

    Code follows sklearn naming/style (e.g. fit(X) to train)

    Parameters
    ----------
    n_hidden : int, optional, default=2
        Number of hidden units.

    dim_hidden : int, optional, default=2
        Each hidden unit can take dim_hidden discrete values.

    max_iter : int, optional
        Maximum number of iterations before ending.

    n_repeat : int, optional
        Repeat several times and take solution with highest TC.

    verbose : int, optional
        The verbosity level. The default, zero, means silent mode. 1 outputs TC(X;Y) as you go
        2 output alpha matrix and MIs as you go.

    seed : integer or numpy.RandomState, optional
        A random number generator instance to define the state of the
        random permutations generator. If an integer is given, it fixes the
        seed. Defaults to the global numpy random number generator.

    Attributes
    ----------
    labels : array, [n_hidden, n_samples]
        Label for each hidden unit for each sample.

    clusters : array, [n_visible]
        Cluster label for each input variable.

    p_y_given_x : array, [n_hidden, n_samples, dim_hidden]
        The distribution of latent factors for each sample.

    alpha : array-like, shape (n_components,)
        Adjacency matrix between input variables and hidden units. In range [0,1].

    mis : array, [n_hidden, n_visible]
        Mutual information between each (visible/observed) variable and hidden unit

    tcs : array, [n_hidden]
        TC(X_Gj;Y_j) for each hidden unit

    tc : float
        Convenience variable = Sum_j tcs[j]

    tc_history : array
        Shows value of TC over the course of learning. Hopefully, it is converging.

    References
    ----------

    [1]     <NAME> and <NAME>. "Discovering Structure in
            High-Dimensional Data Through Correlation Explanation."
            NIPS, 2014. arXiv preprint(arXiv:1406.1222.)

    [2]     <NAME> and <NAME>. "Maximally Informative
            Hierarchical Representations of High-Dimensional Data"
            AISTATS, 2015. arXiv preprint(arXiv:1410.7404.)

    [3]     Pepke and <NAME>, Comprehensive discovery of subsample
            gene expression components by information explanation:
            therapeutic implications in cancer. BMC Medical Genomics, 2017.

    """
    def __init__(self, n_hidden=2, dim_hidden=2,            # Size of representations
                 max_iter=100, n_repeat=1, ram=8., max_samples=1000, n_cpu=1,   # Computational limits
                 eps=1e-5, marginal_description='gaussian', smooth_marginals=False,    # Parameters
                 missing_values=-1, seed=None, verbose=False):

        self.dim_hidden = dim_hidden  # Each hidden factor can take dim_hidden discrete values
        self.n_hidden = n_hidden  # Number of hidden factors to use (Y_1,...Y_m) in paper
        self.missing_values = missing_values  # For a sample value that is unknown

        self.max_iter = max_iter  # Maximum number of updates to run, regardless of convergence
        self.n_repeat = n_repeat  # Run multiple times and take solution with largest TC
        self.ram = ram  # Approximate amount of memory to use in GB
        self.max_samples = max_samples  # The max number of samples to use for estimating MI and unique info
        self.n_cpu = n_cpu  # number of CPU's to use, None will detect number of CPU's
        self.pool = None  # Spin up and close pool of processes in main loop (depending on self.n_cpu)

        self.eps = eps  # Change in TC to signal convergence
        self.smooth_marginals = smooth_marginals  # Less noisy estimation of marginal distributions

        np.random.seed(seed)  # Set seed for deterministic results
        self.verbose = verbose
        if verbose > 0:<commit_msg>This subtle change from multiproc to multiproc.dummy goes from multiprocessing to multithreading. I believe this significantly reduces the amount of memory used because multithreading share the same objects in memory (instead of copying them).<commit_after>"""Maximally Informative Representations using CORrelation EXplanation

Main ideas first described in:
<NAME> and <NAME>. "Maximally Informative
Hierarchical Representations of High-Dimensional Data"
AISTATS, 2015. arXiv preprint(arXiv:1410.7404.)

The Bayesian smoothing option is described in:
Pepke and Ver Steeg, Comprehensive discovery of subsample gene expression components
by information explanation: therapeutic implications in cancer. BMC Medical Genomics, 2017.

Code below written by: Greg <NAME> (<EMAIL>)

License: Apache V2
"""

from __future__ import print_function
import sys
import numpy as np  # Tested with 1.8.0
from os import makedirs
from os import path
from numpy import ma
from scipy.misc import logsumexp  # Tested with 0.13.0
from multiprocessing.dummy import Pool


def unwrap_f(arg):
    """Multiprocessing pool.map requires a top-level function."""
    return Corex.calculate_p_xi_given_y(*arg)

def logsumexp2(z):
    """Multiprocessing pool.map requires a top-level function."""
    return logsumexp(z, axis=2)


class Corex(object):
    """
    Correlation Explanation

    A method to learn a hierarchy of successively more abstract
    representations of complex data that are maximally
    informative about the data. This method is unsupervised,
    requires no assumptions about the data-generating model,
    and scales linearly with the number of variables.

    Code follows sklearn naming/style (e.g. fit(X) to train)

    Parameters
    ----------
    n_hidden : int, optional, default=2
        Number of hidden units.

    dim_hidden : int, optional, default=2
        Each hidden unit can take dim_hidden discrete values.

    max_iter : int, optional
        Maximum number of iterations before ending.

    n_repeat : int, optional
        Repeat several times and take solution with highest TC.

    verbose : int, optional
        The verbosity level. The default, zero, means silent mode. 1 outputs TC(X;Y) as you go
        2 output alpha matrix and MIs as you go.

    seed : integer or numpy.RandomState, optional
        A random number generator instance to define the state of the
        random permutations generator. If an integer is given, it fixes the
        seed. Defaults to the global numpy random number generator.

    Attributes
    ----------
    labels : array, [n_hidden, n_samples]
        Label for each hidden unit for each sample.

    clusters : array, [n_visible]
        Cluster label for each input variable.

    p_y_given_x : array, [n_hidden, n_samples, dim_hidden]
        The distribution of latent factors for each sample.

    alpha : array-like, shape (n_components,)
        Adjacency matrix between input variables and hidden units. In range [0,1].

    mis : array, [n_hidden, n_visible]
        Mutual information between each (visible/observed) variable and hidden unit

    tcs : array, [n_hidden]
        TC(X_Gj;Y_j) for each hidden unit

    tc : float
        Convenience variable = Sum_j tcs[j]

    tc_history : array
        Shows value of TC over the course of learning. Hopefully, it is converging.

    References
    ----------

    [1]     <NAME> and <NAME>. "Discovering Structure in
            High-Dimensional Data Through Correlation Explanation."
            NIPS, 2014. arXiv preprint(arXiv:1406.1222.)

    [2]     <NAME> and <NAME>. "Maximally Informative
            Hierarchical Representations of High-Dimensional Data"
            AISTATS, 2015. arXiv preprint(arXiv:1410.7404.)

    [3]     Pepke and <NAME>, Comprehensive discovery of subsample
            gene expression components by information explanation:
            therapeutic implications in cancer. BMC Medical Genomics, 2017.

    """
    def __init__(self, n_hidden=2, dim_hidden=2,            # Size of representations
                 max_iter=100, n_repeat=1, ram=8., max_samples=10000, n_cpu=1,   # Computational limits
                 eps=1e-5, marginal_description='gaussian', smooth_marginals=False,    # Parameters
                 missing_values=-1, seed=None, verbose=False):

        self.dim_hidden = dim_hidden  # Each hidden factor can take dim_hidden discrete values
        self.n_hidden = n_hidden  # Number of hidden factors to use (Y_1,...Y_m) in paper
        self.missing_values = missing_values  # For a sample value that is unknown

        self.max_iter = max_iter  # Maximum number of updates to run, regardless of convergence
        self.n_repeat = n_repeat  # Run multiple times and take solution with largest TC
        self.ram = ram  # Approximate amount of memory to use in GB
        self.max_samples = max_samples  # The max number of samples to use for estimating MI and unique info
        self.n_cpu = n_cpu  # number of CPU's to use, None will detect number of CPU's
        self.pool = None  # Spin up and close pool of processes in main loop (depending on self.n_cpu)

        self.eps = eps  # Change in TC to signal convergence
        self.smooth_marginals = smooth_marginals  # Less noisy estimation of marginal distributions

        np.random.seed(seed)  # Set seed for deterministic results
        self.verbose = verbose
        if verbose > 0:
{'fraction_non_alphanumeric': 0.06644080557124035, 'fraction_numerical': 0.015151515151515152, 'mean_word_length': 2.9344687152906332, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '386268', 'n_tokens_mistral': 3086, 'n_tokens_neox': 2796, 'n_words': 1185}