<commit_before>from __future__ import absolute_import
import numpy as np
import scipy.spatial.distance as distance


class UniformSampler:
    def __init__(self, n_samples):
        self.n_samples = n_samples

    def generate(self, x, min_sample_size):
        n_elements = x.shape[0]
        for i in range(self.n_samples):
            while True:
                sample = np.random.randint(0, n_elements, size=min_sample_size)
                if np.unique(sample).size == min_sample_size:
                    break
            yield sample


class GaussianLocalSampler:
    def __init__(self, n_samples, sigma):
        self.n_samples = n_samples
        # p(x[i] | x[j]) = exp(-(dist(x[i], x[j])) / sigma)
        self.var = sigma ** 2
        self.distribution = None

    def generate(self, x, min_sample_size):
        n_elements = x.shape[0]
        self.distribution = np.zeros((n_elements,))
        for _ in range(self.n_samples):
            bins = np.cumsum(self.distribution.max() - self.distribution)
            bins /= bins[-1]
            rnd = np.random.random()
            j = np.searchsorted(bins, rnd)
            # j = np.random.randint(0, n_elements)
            dists = distance.cdist(x, np.atleast_2d(x[j, :]), 'euclidean')
            bins = np.cumsum(np.exp(-(dists ** 2) / self.var))
            bins /= bins[-1]
            while True:
                rnd = np.random.random((min_sample_size - 1,))
                sample = np.searchsorted(bins, rnd)
                sample = np.hstack((sample, [j]))
                if np.unique(sample).size == min_sample_size:
                    break
            self.distribution[sample] += 1
            yield sample


def model_distance_generator(model_class, elements, sampler):
    samples = sampler.generate(elements, model_class().min_sample_size)
    for s in samples:
        ms_set = np.take(elements, s, axis=0)
        model = model_class()
        model.fit(ms_set)
        yield model, model.distances(elements)


def inliers_generator(mdg, threshold):
    for model, dist in mdg:
        yield model, dist < threshold


# if __name__ == '__main__':
#     x = np.random.rand(100, 2)
#     sampler = GaussianLocalSampler(0.1)
#     list(sampler.generate(x, 1, 2))<commit_msg>Using <= for inliers check<commit_after>from __future__ import absolute_import
import numpy as np
import scipy.spatial.distance as distance


class UniformSampler:
    def __init__(self, n_samples):
        self.n_samples = n_samples

    def generate(self, x, min_sample_size):
        n_elements = x.shape[0]
        for i in range(self.n_samples):
            while True:
                sample = np.random.randint(0, n_elements, size=min_sample_size)
                if np.unique(sample).size == min_sample_size:
                    break
            yield sample


class GaussianLocalSampler:
    def __init__(self, n_samples, sigma):
        self.n_samples = n_samples
        # p(x[i] | x[j]) = exp(-(dist(x[i], x[j])) / sigma)
        self.var = sigma ** 2
        self.distribution = None

    def generate(self, x, min_sample_size):
        n_elements = x.shape[0]
        self.distribution = np.zeros((n_elements,))
        for _ in range(self.n_samples):
            bins = np.cumsum(self.distribution.max() - self.distribution)
            bins /= bins[-1]
            rnd = np.random.random()
            j = np.searchsorted(bins, rnd)
            # j = np.random.randint(0, n_elements)
            dists = distance.cdist(x, np.atleast_2d(x[j, :]), 'euclidean')
            bins = np.cumsum(np.exp(-(dists ** 2) / self.var))
            bins /= bins[-1]
            while True:
                rnd = np.random.random((min_sample_size - 1,))
                sample = np.searchsorted(bins, rnd)
                sample = np.hstack((sample, [j]))
                if np.unique(sample).size == min_sample_size:
                    break
            self.distribution[sample] += 1
            yield sample


def model_distance_generator(model_class, elements, sampler):
    samples = sampler.generate(elements, model_class().min_sample_size)
    for s in samples:
        ms_set = np.take(elements, s, axis=0)
        model = model_class()
        model.fit(ms_set)
        yield model, model.distances(elements)


def inliers_generator(mdg, threshold):
    for model, dist in mdg:
        yield model, dist <= threshold


# if __name__ == '__main__':
#     x = np.random.rand(100, 2)
#     sampler = GaussianLocalSampler(0.1)
#     list(sampler.generate(x, 1, 2))
{'fraction_non_alphanumeric': 0.11771201413427562, 'fraction_numerical': 0.0088339222614841, 'mean_word_length': 2.1342560553633216, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5732882', 'n_tokens_mistral': 1505, 'n_tokens_neox': 1407, 'n_words': 314}