<commit_before># Licensed under an MIT open source license - see LICENSE

'''

The density PDF as described by Kowal et al. (2007)

'''

import numpy as np
from scipy.stats import ks_2samp, anderson_ksamp

from ..stats_utils import hellinger, standardize, common_histogram_bins


class PDF(object):
    '''
    Create the PDF of a given array.
    '''
    def __init__(self, img, min_val=0.0, bins=None, weights=None,
                 use_standardized=False):
        super(PDF, self).__init__()

        self.img = img

        # We want to remove NaNs and value below the threshold.
        self.data = img[np.isfinite(img)]
        self.data = self.data[self.data > min_val]

        # Do the same for the weights, then apply weights to the data.
        if weights is not None:
            self.weights = weights[np.isfinite(img)]
            self.weights = self.weights[self.data > min_val]

            self.data *= self.weights

        if use_standardized:
            self.data = standardize(self.data)

        self._bins = bins

    def make_pdf(self, bins=None):
        '''
        Create the PDF.
        '''

        if bins is not None:
            self._bins = bins

        # If the number of bins is not given, use sqrt of data length.
        if self.bins is None:
            self._bins = np.sqrt(self.data.shape[0])
            self._bins = int(np.round(self.bins))

        norm_weights = np.ones_like(self.data) / self.data.shape[0]

        self._pdf, bin_edges = np.histogram(self.data, bins=self.bins,
                                            density=False,
                                            weights=norm_weights)

        self._bins = (bin_edges[:-1] + bin_edges[1:]) / 2

        return self

    @property
    def pdf(self):
        return self._pdf

    @property
    def bins(self):
        return self._bins

    def make_ecdf(self):
        '''
        Create the ECDF.
        '''

        self._ecdf = np.cumsum(np.sort(self.data.ravel())) / np.sum(self.data)

        return self

    @property
    def ecdf(self):
        return self._ecdf

    def run(self, verbose=False, bins=None):
        '''
        Run the whole thing.
        '''

        self.make_pdf(bins=bins)
        self.make_ecdf()

        if verbose:
            import matplotlib.pyplot as p
            # PDF
            p.subplot(131)
            p.loglog(self.bins[self.pdf > 0], self.pdf[self.pdf > 0], 'b-')
            p.grid(True)
            p.xlabel(r"$\Sigma/\overline{\Sigma}$")
            p.ylabel("PDF")

            # ECDF
            p.subplot(132)
            p.semilogx(np.sort(self.data.ravel()), self.ecdf, 'b-')
            p.grid(True)
            p.xlabel(r"$\Sigma/\overline{\Sigma}$")
            p.ylabel("ECDF")

            # Array representation.
            p.subplot(133)
            if self.img.ndim == 1:
                p.plot(self.img, 'b-')
            elif self.img.ndim == 2:
                p.imshow(self.img, origin="lower", interpolation="nearest",
                         cmap="binary")
            elif self.img.ndim == 3:
                p.imshow(np.nansum(self.img, axis=0), origin="lower",
                         interpolation="nearest", cmap="binary")
            else:
                print("Visual representation works only up to 3D.")

            p.show()

        return self


class PDF_Distance(object):
    '''
    Calculate the distance between two arrays using their PDFs.

    Parameters
    ----------
    img1 : numpy.ndarray
        Array (1-3D).
    img2 : numpy.ndarray
        Array (1-3D).
    min_val1 : float, optional
        Minimum value to keep in img1
    min_val2 : float, optional
        Minimum value to keep in img2
    weights1 : numpy.ndarray, optional
        Weights to be used with img1
    weights2 : numpy.ndarray, optional
        Weights to be used with img2
    '''
    def __init__(self, img1, img2, min_val1=0.0, min_val2=0.0,
                 weights1=None, weights2=None):
        super(PDF_Distance, self).__init__()

        self.img1 = img1
        self.img2 = img2

        if weights1 is None:
            weights1 = np.ones_like(img1)

        if weights2 is None:
            weights2 = np.ones_like(img2)

        self.PDF1 = PDF(self.img1, use_standardized=True)

        self.PDF2 = PDF(self.img2, use_standardized=True)

        self.bins = common_histogram_bins(self.PDF1.data, self.PDF2.data)

        # Feed the common set of bins to be used in the PDFs
        self.PDF1.run(verbose=False, bins=self.bins)
        self.PDF2.run(verbose=False, bins=self.bins)

    def compute_hellinger_distance(self):
        '''
        Computes the Hellinger Distance between the two PDFs.
        '''

        self.hellinger_distance = hellinger(self.PDF1.pdf, self.PDF2.pdf)

        return self

    def compute_ks_distance(self):
        '''<commit_msg>Pass the min_val and weights<commit_after># Licensed under an MIT open source license - see LICENSE

'''

The density PDF as described by Kowal et al. (2007)

'''

import numpy as np
from scipy.stats import ks_2samp, anderson_ksamp

from ..stats_utils import hellinger, standardize, common_histogram_bins


class PDF(object):
    '''
    Create the PDF of a given array.
    '''
    def __init__(self, img, min_val=0.0, bins=None, weights=None,
                 use_standardized=False):
        super(PDF, self).__init__()

        self.img = img

        # We want to remove NaNs and value below the threshold.
        self.data = img[np.isfinite(img)]
        self.data = self.data[self.data > min_val]

        # Do the same for the weights, then apply weights to the data.
        if weights is not None:
            self.weights = weights[np.isfinite(img)]
            self.weights = self.weights[self.data > min_val]

            self.data *= self.weights

        if use_standardized:
            self.data = standardize(self.data)

        self._bins = bins

    def make_pdf(self, bins=None):
        '''
        Create the PDF.
        '''

        if bins is not None:
            self._bins = bins

        # If the number of bins is not given, use sqrt of data length.
        if self.bins is None:
            self._bins = np.sqrt(self.data.shape[0])
            self._bins = int(np.round(self.bins))

        norm_weights = np.ones_like(self.data) / self.data.shape[0]

        self._pdf, bin_edges = np.histogram(self.data, bins=self.bins,
                                            density=False,
                                            weights=norm_weights)

        self._bins = (bin_edges[:-1] + bin_edges[1:]) / 2

        return self

    @property
    def pdf(self):
        return self._pdf

    @property
    def bins(self):
        return self._bins

    def make_ecdf(self):
        '''
        Create the ECDF.
        '''

        self._ecdf = np.cumsum(np.sort(self.data.ravel())) / np.sum(self.data)

        return self

    @property
    def ecdf(self):
        return self._ecdf

    def run(self, verbose=False, bins=None):
        '''
        Run the whole thing.
        '''

        self.make_pdf(bins=bins)
        self.make_ecdf()

        if verbose:
            import matplotlib.pyplot as p
            # PDF
            p.subplot(131)
            p.loglog(self.bins[self.pdf > 0], self.pdf[self.pdf > 0], 'b-')
            p.grid(True)
            p.xlabel(r"$\Sigma/\overline{\Sigma}$")
            p.ylabel("PDF")

            # ECDF
            p.subplot(132)
            p.semilogx(np.sort(self.data.ravel()), self.ecdf, 'b-')
            p.grid(True)
            p.xlabel(r"$\Sigma/\overline{\Sigma}$")
            p.ylabel("ECDF")

            # Array representation.
            p.subplot(133)
            if self.img.ndim == 1:
                p.plot(self.img, 'b-')
            elif self.img.ndim == 2:
                p.imshow(self.img, origin="lower", interpolation="nearest",
                         cmap="binary")
            elif self.img.ndim == 3:
                p.imshow(np.nansum(self.img, axis=0), origin="lower",
                         interpolation="nearest", cmap="binary")
            else:
                print("Visual representation works only up to 3D.")

            p.show()

        return self


class PDF_Distance(object):
    '''
    Calculate the distance between two arrays using their PDFs.

    Parameters
    ----------
    img1 : numpy.ndarray
        Array (1-3D).
    img2 : numpy.ndarray
        Array (1-3D).
    min_val1 : float, optional
        Minimum value to keep in img1
    min_val2 : float, optional
        Minimum value to keep in img2
    weights1 : numpy.ndarray, optional
        Weights to be used with img1
    weights2 : numpy.ndarray, optional
        Weights to be used with img2
    '''
    def __init__(self, img1, img2, min_val1=0.0, min_val2=0.0,
                 weights1=None, weights2=None):
        super(PDF_Distance, self).__init__()

        self.img1 = img1
        self.img2 = img2

        self.PDF1 = PDF(self.img1, min_val=min_val1, use_standardized=True,
                        weights=weights1)

        self.PDF2 = PDF(self.img2, min_val=min_val2, use_standardized=True,
                        weights=weights2)

        self.bins = common_histogram_bins(self.PDF1.data, self.PDF2.data)

        # Feed the common set of bins to be used in the PDFs
        self.PDF1.run(verbose=False, bins=self.bins)
        self.PDF2.run(verbose=False, bins=self.bins)

    def compute_hellinger_distance(self):
        '''
        Computes the Hellinger Distance between the two PDFs.
        '''

        self.hellinger_distance = hellinger(self.PDF1.pdf, self.PDF2.pdf)

        return self

    def compute_ks_distance(self):
        '''
{'fraction_non_alphanumeric': 0.10991268618387262, 'fraction_numerical': 0.014586543400102722, 'mean_word_length': 1.7325287678922257, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '809485', 'n_tokens_mistral': 3123, 'n_tokens_neox': 2912, 'n_words': 728}