<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Author: <NAME> <<EMAIL>>
#         <NAME> <<EMAIL>>
#
# License: Simplified BSD

import numpy as np
from scipy import stats, sparse, ndimage

from .parametric import f_oneway


def _find_clusters(x, threshold, tail=0, connectivity=None):
    """For a given 1d-array (test statistic), find all clusters which
    are above/below a certain threshold. Returns a list of 2-tuples.

    Parameters
    ----------
    x: 1D array
        Data
    threshold: float
        Where to threshold the statistic
    tail : -1 | 0 | 1
        Type of comparison

    Returns
    -------
    clusters: list of slices or list of arrays (boolean masks)
        We use slices for 1D signals and mask to multidimensional
        arrays.

    sums: array
        Sum of x values in clusters
    """
    if not tail in [-1, 0, 1]:
        raise ValueError('invalid tail parameter')

    x = np.asanyarray(x)

    if tail == -1:
        x_in = x < threshold
    elif tail == 1:
        x_in = x > threshold
    else:
        x_in = np.abs(x) > threshold

    labels, n_labels = ndimage.label(x_in)

    if connectivity is None:
        if x.ndim == 1:
            clusters = ndimage.find_objects(labels, n_labels)
            sums = ndimage.measurements.sum(x, labels,
                                            index=range(1, n_labels + 1))
        else:
            clusters = list()
            sums = np.empty(n_labels)
            for l in range(1, n_labels + 1):
                c = labels == l
                clusters.append(c)
                sums[l - 1] = np.sum(x[c])
    else:
        if x.ndim > 1:
            raise Exception("Data should be 1D when using a connectivity "
                            "to define clusters.")
        from scikits.learn.utils._csgraph import cs_graph_components
        mask = np.logical_and(x_in[connectivity.row], x_in[connectivity.col])
        if np.sum(mask) == 0:
            return [], np.empty(0)
        connectivity = sparse.coo_matrix((connectivity.data[mask],
                                         (connectivity.row[mask],
                                          connectivity.col[mask])))
        _, components = cs_graph_components(connectivity)
        labels = np.unique(components)
        clusters = list()
        sums = list()
        for l in labels:
            c = (components == l)
            if np.any(x_in[c]):
                clusters.append(c)
                sums.append(np.sum(x[c]))
        sums = np.array(sums)
    return clusters, sums


def _pval_from_histogram(T, H0, tail):
    """Get p-values from stats values given an H0 distribution

    For each stat compute a p-value as percentile of its statistics
    within all statistics in surrogate data
    """
    if not tail in [-1, 0, 1]:
        raise ValueError('invalid tail parameter')

    # from pct to fraction
    if tail == -1:  # up tail
        pval = np.array([np.mean(H0 <= t) for t in T])
    elif tail == 1:  # low tail
        pval = np.array([np.mean(H0 >= t) for t in T])
    elif tail == 0:  # both tails
        pval = np.array([np.mean(H0 >= abs(t)) for t in T])
        pval += np.array([np.mean(H0 <= -abs(t)) for t in T])

    return pval


def permutation_cluster_test(X, stat_fun=f_oneway, threshold=1.67,
                             n_permutations=1000, tail=0,
                             connectivity=None):
    """Cluster-level statistical permutation test

    For a list of 2d-arrays of data, e.g. power values, calculate some
    statistics for each timepoint (dim 1) over groups.  Do a cluster
    analysis with permutation test for calculating corrected p-values.
    Randomized data are generated with random partitions of the data.<commit_msg>FIX : bug fix in cluster_level with connectivity + FIX in pval computation<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Author: <NAME> <<EMAIL>>
#         <NAME> <<EMAIL>>
#
# License: Simplified BSD

import numpy as np
from scipy import stats, sparse, ndimage

from .parametric import f_oneway


def _find_clusters(x, threshold, tail=0, connectivity=None):
    """For a given 1d-array (test statistic), find all clusters which
    are above/below a certain threshold. Returns a list of 2-tuples.

    Parameters
    ----------
    x: 1D array
        Data
    threshold: float
        Where to threshold the statistic
    tail : -1 | 0 | 1
        Type of comparison

    Returns
    -------
    clusters: list of slices or list of arrays (boolean masks)
        We use slices for 1D signals and mask to multidimensional
        arrays.

    sums: array
        Sum of x values in clusters
    """
    if not tail in [-1, 0, 1]:
        raise ValueError('invalid tail parameter')

    x = np.asanyarray(x)

    if tail == -1:
        x_in = x < threshold
    elif tail == 1:
        x_in = x > threshold
    else:
        x_in = np.abs(x) > threshold

    labels, n_labels = ndimage.label(x_in)

    if connectivity is None:
        if x.ndim == 1:
            clusters = ndimage.find_objects(labels, n_labels)
            sums = ndimage.measurements.sum(x, labels,
                                            index=range(1, n_labels + 1))
        else:
            clusters = list()
            sums = np.empty(n_labels)
            for l in range(1, n_labels + 1):
                c = labels == l
                clusters.append(c)
                sums[l - 1] = np.sum(x[c])
    else:
        if x.ndim > 1:
            raise Exception("Data should be 1D when using a connectivity "
                            "to define clusters.")
        from scikits.learn.utils._csgraph import cs_graph_components
        mask = np.logical_and(x_in[connectivity.row], x_in[connectivity.col])
        if np.sum(mask) == 0:
            return [], np.empty(0)
        mask = np.logical_and(x_in[connectivity.row], x_in[connectivity.col])
        connectivity = sparse.coo_matrix((connectivity.data[mask],
                                         (connectivity.row[mask],
                                          connectivity.col[mask])),
                                          shape=connectivity.shape)
        idx = np.where(x_in)[0]
        data = np.ones(len(idx), dtype=connectivity.data.dtype)
        connectivity.row = np.concatenate((connectivity.row, idx))
        connectivity.col = np.concatenate((connectivity.col, idx))
        connectivity.data = np.concatenate((connectivity.data, data))
        _, components = cs_graph_components(connectivity)
        labels = np.unique(components)
        clusters = list()
        sums = list()
        for l in labels:
            c = (components == l)
            if np.any(x_in[c]):
                clusters.append(c)
                sums.append(np.sum(x[c]))
        sums = np.array(sums)
    return clusters, sums


def _pval_from_histogram(T, H0, tail):
    """Get p-values from stats values given an H0 distribution

    For each stat compute a p-value as percentile of its statistics
    within all statistics in surrogate data
    """
    if not tail in [-1, 0, 1]:
        raise ValueError('invalid tail parameter')

    # from pct to fraction
    if tail == -1:  # up tail
        pval = np.array([np.sum(H0 <= t) for t in T])
    elif tail == 1:  # low tail
        pval = np.array([np.sum(H0 >= t) for t in T])
    elif tail == 0:  # both tails
        pval = np.array([np.sum(H0 >= abs(t)) for t in T])
        pval += np.array([np.sum(H0 <= -abs(t)) for t in T])

    pval = (pval + 1.0) / (H0.size + 1.0)  # the init data is one resampling
    return pval


def permutation_cluster_test(X, stat_fun=f_oneway, threshold=1.67,
                             n_permutations=1000, tail=0,
                             connectivity=None):
    """Cluster-level statistical permutation test

    For a list of 2d-arrays of data, e.g. power values, calculate some
    statistics for each timepoint (dim 1) over groups.  Do a cluster
    analysis with permutation test for calculating corrected p-values.
    Randomized data are generated with random partitions of the data.
{'fraction_non_alphanumeric': 0.10451628825271471, 'fraction_numerical': 0.01209279368213228, 'mean_word_length': 1.8936094252052837, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6582088', 'n_tokens_mistral': 2494, 'n_tokens_neox': 2367, 'n_words': 766}