<commit_before>"""
Functions for transforming flow cytometer data to MEF units.

"""

import os
import functools
import collections

import numpy as np
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from sklearn.mixture import GMM

import FlowCal.plot
import FlowCal.transform
import FlowCal.stats

# Use default colors from palettable if available
try:
    import palettable
except ImportError, e:
    standard_curve_colors = ['b', 'g', 'r']
else:
    standard_curve_colors = \
        palettable.colorbrewer.qualitative.Paired_12.mpl_colors[1::2]

def clustering_gmm(data,
                   n_clusters,
                   tol=1e-7,
                   min_covar=1e-2,
                   scale='log'):
    """
    Find clusters in an array using Gaussian Mixture Models (GMM).

    Parameters
    ----------
    data : array_like
        NxD array to cluster.
    n_clusters : int
        Number of clusters to find.
    tol : float, optional
        Tolerance for convergence of GMM method. Passed directly to
        ``scikit-learn``'s GMM.
    min_covar : float, optional
        Minimum covariance. Passed directly to ``scikit-learn``'s GMM.
    scale : str, optional
        Rescaling applied to `data` before performing clustering. If
        ``linear``, no rescaling is applied. If ``log``, base-10 logarithm
        is applied.

    Returns
    -------
    labels : array
        Nx1 array with labels for each element in `data`, assigning
        ``data[i]`` to cluster ``labels[i]``.

    Notes
    -----
    GMM finds clusters by fitting a linear combination of `n_clusters`
    Gaussian probability density functions (pdf) to `data` using
    Expectation Maximization (EM).

    GMM can be fairly sensitive to the initial parameter choice. To
    generate a reasonable set of initial conditions, `clustering_gmm`
    first divides all samples in `data` into `n_clusters` groups of the
    same size based on their Euclidean distance to the origin. Then, for
    each group, the 50% samples farther away from the mean are discarded.
    The mean and covariance are calculated from the remaining samples of
    each group, and used as initial conditions for the GMM EM algorithm.

    `clustering_gmm` internally uses `GMM` from the ``scikit-learn``
    library, with full covariance matrices for each cluster and a fixed,
    uniform set of weights. This means that `clustering_gmm` implicitly
    assumes that all bead subpopulations have roughly the same number of
    events. For more information, consult ``scikit-learn``'s documentation.

    """
    # Check that ``scale`` is supported
    if scale!='linear' and scale!='log':
        raise ValueError("scale {} not supported".format(scale))

    # Rescale if necessary
    if scale=='log':
        # Copy events
        data = data.copy()
        # Log scale requires saturating negative and zero-valued events
        data[data < 1] = 1.
        # Rescale
        data = np.log10(data)*256

    ###
    # Parameter initialization
    ###
    weights = np.tile(1.0 / n_clusters, n_clusters)
    means = []
    covars = []

    # Get distance and sort based on it
    dist = np.sum(data**2., axis=1)
    sorted_idx = np.argsort(dist)

    # Expected number of elements per cluster
    n_per_cluster = data.shape[0]/float(n_clusters)

    # Get means and covariances per cluster
    # We will just use a fraction of ``1 - discard_frac`` of the data.
    # Data at the edges that actually corresponds to another cluster can
    # really mess up the final result.
    discard_frac = 0.5
    for i in range(n_clusters):
        il = int((i + discard_frac/2)*n_per_cluster)
        ih = int((i + 1 - discard_frac/2)*n_per_cluster)
        sorted_idx_cluster = sorted_idx[il:ih]
        data_cluster = data[sorted_idx_cluster]
        means.append(np.mean(data_cluster, axis=0))
        if data.shape[1] == 1:
            covars.append(np.cov(data_cluster.T).reshape(1,1))
        else:<commit_msg>Changed log scaling in mef.clustering_gmm.<commit_after>"""
Functions for transforming flow cytometer data to MEF units.

"""

import os
import functools
import collections

import numpy as np
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from sklearn.mixture import GMM

import FlowCal.plot
import FlowCal.transform
import FlowCal.stats

# Use default colors from palettable if available
try:
    import palettable
except ImportError, e:
    standard_curve_colors = ['b', 'g', 'r']
else:
    standard_curve_colors = \
        palettable.colorbrewer.qualitative.Paired_12.mpl_colors[1::2]

def clustering_gmm(data,
                   n_clusters,
                   tol=1e-7,
                   min_covar=5e-5,
                   scale='log'):
    """
    Find clusters in an array using Gaussian Mixture Models (GMM).

    Parameters
    ----------
    data : array_like
        NxD array to cluster.
    n_clusters : int
        Number of clusters to find.
    tol : float, optional
        Tolerance for convergence of GMM method. Passed directly to
        ``scikit-learn``'s GMM.
    min_covar : float, optional
        Minimum covariance. Passed directly to ``scikit-learn``'s GMM.
    scale : str, optional
        Rescaling applied to `data` before performing clustering. If
        ``linear``, no rescaling is applied. If ``log``, base-10 logarithm
        is applied.

    Returns
    -------
    labels : array
        Nx1 array with labels for each element in `data`, assigning
        ``data[i]`` to cluster ``labels[i]``.

    Notes
    -----
    GMM finds clusters by fitting a linear combination of `n_clusters`
    Gaussian probability density functions (pdf) to `data` using
    Expectation Maximization (EM).

    GMM can be fairly sensitive to the initial parameter choice. To
    generate a reasonable set of initial conditions, `clustering_gmm`
    first divides all samples in `data` into `n_clusters` groups of the
    same size based on their Euclidean distance to the origin. Then, for
    each group, the 50% samples farther away from the mean are discarded.
    The mean and covariance are calculated from the remaining samples of
    each group, and used as initial conditions for the GMM EM algorithm.

    `clustering_gmm` internally uses `GMM` from the ``scikit-learn``
    library, with full covariance matrices for each cluster and a fixed,
    uniform set of weights. This means that `clustering_gmm` implicitly
    assumes that all bead subpopulations have roughly the same number of
    events. For more information, consult ``scikit-learn``'s documentation.

    """
    # Check that ``scale`` is supported
    if scale!='linear' and scale!='log':
        raise ValueError("scale {} not supported".format(scale))

    # Rescale if necessary
    if scale=='log':
        # Copy events
        data = data.copy()
        # Log scale requires saturating negative and zero-valued events
        data[data < 1] = 1.
        # Rescale
        data = np.log10(data)

    ###
    # Parameter initialization
    ###
    weights = np.tile(1.0 / n_clusters, n_clusters)
    means = []
    covars = []

    # Get distance and sort based on it
    dist = np.sum(data**2., axis=1)
    sorted_idx = np.argsort(dist)

    # Expected number of elements per cluster
    n_per_cluster = data.shape[0]/float(n_clusters)

    # Get means and covariances per cluster
    # We will just use a fraction of ``1 - discard_frac`` of the data.
    # Data at the edges that actually corresponds to another cluster can
    # really mess up the final result.
    discard_frac = 0.5
    for i in range(n_clusters):
        il = int((i + discard_frac/2)*n_per_cluster)
        ih = int((i + 1 - discard_frac/2)*n_per_cluster)
        sorted_idx_cluster = sorted_idx[il:ih]
        data_cluster = data[sorted_idx_cluster]
        means.append(np.mean(data_cluster, axis=0))
        if data.shape[1] == 1:
            covars.append(np.cov(data_cluster.T).reshape(1,1))
        else:
{'fraction_non_alphanumeric': 0.08726680856391636, 'fraction_numerical': 0.008639038437460875, 'mean_word_length': 2.689607390300231, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1013296', 'n_tokens_mistral': 2394, 'n_tokens_neox': 2260, 'n_words': 839}