<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Author: <NAME> <<EMAIL>>
#         <NAME> <<EMAIL>>
#
# License: Simplified BSD

import numpy as np
from scipy import ndimage
from scipy.stats import percentileofscore, ttest_1samp

from .parametric import f_oneway


def _find_clusters(x, threshold, tail=0):
    """For a given 1d-array (test statistic), find all clusters which
    are above/below a certain threshold. Returns a list of 2-tuples.

    Parameters
    ----------
    x: 1D array
        Data
    threshold: float
        Where to threshold the statistic
    tail : -1 | 0 | 1
        Type of comparison

    Returns
    -------
    clusters: list of slices or list of arrays (boolean masks)
        We use slices for 1D signals and mask to multidimensional
        arrays.

    sums: array
        Sum of x values in clusters
    """
    if not tail in [-1, 0, 1]:
        raise ValueError('invalid tail parameter')

    x = np.asanyarray(x)

    if tail == -1:
        x_in = x < threshold
    elif tail == 1:
        x_in = x > threshold
    else:
        x_in = np.abs(x) > threshold

    labels, n_labels = ndimage.label(x_in)

    if x.ndim == 1:
        clusters = ndimage.find_objects(labels, n_labels)
        sums = ndimage.measurements.sum(x, labels, index=range(1, n_labels+1))
    else:
        clusters = list()
        sums = np.empty(n_labels)
        for l in range(1, n_labels+1):
            c = labels == l
            clusters.append(c)
            sums[l-1] = np.sum(x[c])

    return clusters, sums


def _pval_from_histogram(T, H0, tail):
    """Get p-values from stats values given an H0 distribution

    For each stat compute a p-value as percentile of its statistics
    within all statistics in surrogate data
    """
    if not tail in [-1, 0, 1]:
        raise ValueError('invalid tail parameter')

    pval = np.array([percentileofscore(H0, t) for t in T])

    # from pct to fraction
    if tail == -1: # up tail
        pval =  pval / 100.0
    elif tail == 1: # low tail
        pval = (100.0 - pval) / 100.0
    elif tail == 0: # both tails
        pval = 100.0 - pval
        pval += np.array([percentileofscore(H0, -t) for t in T])
<commit_msg>more bug fix in permutation_cluster_t_test<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Author: <NAME> <<EMAIL>>
#         <NAME> <<EMAIL>>
#
# License: Simplified BSD

import numpy as np
from scipy import ndimage
from scipy.stats import percentileofscore, ttest_1samp

from .parametric import f_oneway


def _find_clusters(x, threshold, tail=0):
    """For a given 1d-array (test statistic), find all clusters which
    are above/below a certain threshold. Returns a list of 2-tuples.

    Parameters
    ----------
    x: 1D array
        Data
    threshold: float
        Where to threshold the statistic
    tail : -1 | 0 | 1
        Type of comparison

    Returns
    -------
    clusters: list of slices or list of arrays (boolean masks)
        We use slices for 1D signals and mask to multidimensional
        arrays.

    sums: array
        Sum of x values in clusters
    """
    if not tail in [-1, 0, 1]:
        raise ValueError('invalid tail parameter')

    x = np.asanyarray(x)

    if tail == -1:
        x_in = x < threshold
    elif tail == 1:
        x_in = x > threshold
    else:
        x_in = np.abs(x) > threshold

    labels, n_labels = ndimage.label(x_in)

    if x.ndim == 1:
        clusters = ndimage.find_objects(labels, n_labels)
        sums = ndimage.measurements.sum(x, labels, index=range(1, n_labels+1))
    else:
        clusters = list()
        sums = np.empty(n_labels)
        for l in range(1, n_labels+1):
            c = labels == l
            clusters.append(c)
            sums[l-1] = np.sum(x[c])

    return clusters, sums


def _pval_from_histogram(T, H0, tail):
    """Get p-values from stats values given an H0 distribution

    For each stat compute a p-value as percentile of its statistics
    within all statistics in surrogate data
    """
    if not tail in [-1, 0, 1]:
        raise ValueError('invalid tail parameter')

    pval = np.array([percentileofscore(H0, t) for t in T])

    # from pct to fraction
    if tail == -1: # up tail
        pval =  pval / 100.0
    elif tail == 1: # low tail
        pval = (100.0 - pval) / 100.0
    elif tail == 0: # both tails
        pval = 100.0 - pval
        pval += np.array([percentileofscore(H0, -t) for t in T])
        pval /= 100.0

{'fraction_non_alphanumeric': 0.10887547507265817, 'fraction_numerical': 0.02190923317683881, 'mean_word_length': 2.1330532212885154, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1849156', 'n_tokens_mistral': 1527, 'n_tokens_neox': 1435, 'n_words': 502}