<commit_before>"""Methods for matching the empirical probability distribution of two data sets."""

import numpy as np
from scipy import interpolate as sip
from scipy import optimize as sop

def compute_empirical_cdf(bin_edges, hist):
    """Compute an empirical cumulative distribution function from the given
    histogram.

    Parameters
    ----------
    bin_edges : array_like
        Coordinates of left edges of the histogram bins.
    hist : array_like
        Histogram counts for each bin.

    Returns
    -------
    out : ndarray
        CDF values corresponding to the bin edges.

    """
    cdf = []
    xs = 0.0

    for x,h in zip(zip(bin_edges[:-1], bin_edges[1:]), hist):
        cdf.append(xs)
        xs += (x[1] - x[0]) * h

    cdf.append(xs)
    cdf = np.array(cdf) / xs

    return cdf

def nonparam_match_empirical_cdf(R, R_trg):
    """Matches the empirical CDF of the initial array with the empirical CDF
    of a target array. Initial ranks are conserved, but empirical distribution
    matches the target one. Zero-pixels (i.e. pixels having the minimum value)
    in the initial array are conserved.

    Parameters
    ----------
    R : array_like
        The initial array whose CDF is to be matched with the target.
    R_trg : array_like
        The target array.

    Returns
    -------
    out : array_like
        The matched array.

    """
    if R.size != R_trg.size:
        raise ValueError("the input arrays must have the same size")
    if np.any(~np.isfinite(R)):
        raise ValueError("initial array contains non-finite values")
    if np.any(~np.isfinite(R_trg)):
        raise ValueError("target array contains non-finite values")

    # zeros in initial array
    zvalue = R.min()
    idxzeros = R == zvalue

    # zeros in the target array
    zvalue_trg = R_trg.min()

    # adjust the fraction of rain in target distribution if the number of zeros
    # is greater than in the initial array
    if np.sum(R_trg > zvalue_trg) > np.sum(R > zvalue):
        war = np.sum(R > zvalue)/R.size
        p = np.percentile(R_trg, 100*(1 - war))
        R_trg = R_trg.copy()
        R_trg[R_trg < p] = zvalue_trg

    # flatten the arrays
    arrayshape = R.shape
    R_trg = R_trg.flatten()
    R = R.flatten()

    # rank target values
    order = R_trg.argsort()
    ranked = R_trg[order]

    # rank initial values order
    orderin = R.argsort()
    ranks = np.empty(len(R), int)
    ranks[orderin] = np.arange(len(R))

    # get ranked values from target and rearrange with the initial order
    R = ranked[ranks]

    # reshape to the original array dimensions
    R = R.reshape(arrayshape)<commit_msg>Fix incorrect comment line<commit_after>"""Methods for matching the empirical probability distribution of two data sets."""

import numpy as np
from scipy import interpolate as sip
from scipy import optimize as sop

def compute_empirical_cdf(bin_edges, hist):
    """Compute an empirical cumulative distribution function from the given
    histogram.

    Parameters
    ----------
    bin_edges : array_like
        Coordinates of left edges of the histogram bins.
    hist : array_like
        Histogram counts for each bin.

    Returns
    -------
    out : ndarray
        CDF values corresponding to the bin edges.

    """
    cdf = []
    xs = 0.0

    for x,h in zip(zip(bin_edges[:-1], bin_edges[1:]), hist):
        cdf.append(xs)
        xs += (x[1] - x[0]) * h

    cdf.append(xs)
    cdf = np.array(cdf) / xs

    return cdf

def nonparam_match_empirical_cdf(R, R_trg):
    """Matches the empirical CDF of the initial array with the empirical CDF
    of a target array. Initial ranks are conserved, but empirical distribution
    matches the target one. Zero-pixels (i.e. pixels having the minimum value)
    in the initial array are conserved.

    Parameters
    ----------
    R : array_like
        The initial array whose CDF is to be matched with the target.
    R_trg : array_like
        The target array.

    Returns
    -------
    out : array_like
        The matched array.

    """
    if R.size != R_trg.size:
        raise ValueError("the input arrays must have the same size")
    if np.any(~np.isfinite(R)):
        raise ValueError("initial array contains non-finite values")
    if np.any(~np.isfinite(R_trg)):
        raise ValueError("target array contains non-finite values")

    # zeros in initial array
    zvalue = R.min()
    idxzeros = R == zvalue

    # zeros in the target array
    zvalue_trg = R_trg.min()

    # adjust the fraction of rain in target distribution if the number of
    # nonzeros is greater than in the initial array
    if np.sum(R_trg > zvalue_trg) > np.sum(R > zvalue):
        war = np.sum(R > zvalue)/R.size
        p = np.percentile(R_trg, 100*(1 - war))
        R_trg = R_trg.copy()
        R_trg[R_trg < p] = zvalue_trg

    # flatten the arrays
    arrayshape = R.shape
    R_trg = R_trg.flatten()
    R = R.flatten()

    # rank target values
    order = R_trg.argsort()
    ranked = R_trg[order]

    # rank initial values order
    orderin = R.argsort()
    ranks = np.empty(len(R), int)
    ranks[orderin] = np.arange(len(R))

    # get ranked values from target and rearrange with the initial order
    R = ranked[ranks]

    # reshape to the original array dimensions
    R = R.reshape(arrayshape)
{'fraction_non_alphanumeric': 0.09711286089238845, 'fraction_numerical': 0.0037495313085864268, 'mean_word_length': 2.42866323907455, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2542239', 'n_tokens_mistral': 1725, 'n_tokens_neox': 1658, 'n_words': 574}