<commit_before>from __future__ import division
import warnings

import scipy.sparse as sp
import numpy as np
from numpy.testing import assert_allclose

from sklearn.metrics import euclidean_distances
from sklearn.random_projection import (johnson_lindenstrauss_min_dim,
                                       gaussian_random_matrix,
                                       bernouilli_random_matrix,
                                       BernouilliRandomProjection,
                                       GaussianRandomProjection,
                                       BaseRandomProjection,
                                       )

from sklearn.utils.testing import (assert_less,
                                   assert_raises, assert_raise_message,
                                   assert_array_equal,
                                   assert_equal, assert_almost_equal,
                                   assert_in,
                                   )


sparse_random_projection = [BernouilliRandomProjection]
dense_random_projection = [GaussianRandomProjection]
all_random_projection = set(sparse_random_projection + dense_random_projection)


# Make some random data with uniformly located non zero entries with
# gaussian distributed values
def make_sparse_random_data(n_samples, n_features, n_nonzeros):
    rng = np.random.RandomState(0)
    data_coo = sp.coo_matrix(
        (rng.randn(n_nonzeros),
        (rng.randint(n_samples, size=n_nonzeros),
         rng.randint(n_features, size=n_nonzeros))),
        shape=(n_samples, n_features))
    return data_coo.toarray(), data_coo.tocsr()


n_samples, n_features = (10, 1000)
n_nonzeros = n_samples * n_features / 100.
data, data_csr = make_sparse_random_data(n_samples, n_features, n_nonzeros)


###############################################################################
# test on JL lemma
###############################################################################
def test_invalid_jl_domain():
    assert_raises(ValueError, johnson_lindenstrauss_min_dim, 100, 1.1)
    assert_raises(ValueError, johnson_lindenstrauss_min_dim, 100, 0.0)
    assert_raises(ValueError, johnson_lindenstrauss_min_dim, 100, -0.1)


###############################################################################
# tests on random matrix generation
###############################################################################
def check_input_size_random_matrix(random_matrix):
    assert_raises(ValueError, random_matrix, 0, 0)
    assert_raises(ValueError, random_matrix, -1, 1)
    assert_raises(ValueError, random_matrix, 1, -1)
    assert_raises(ValueError, random_matrix, 1, 0)
    assert_raises(ValueError, random_matrix, -1, 0)


def check_size_generated(random_matrix):
    assert_equal(gaussian_random_matrix(1, 5).shape, (1, 5))
    assert_equal(gaussian_random_matrix(5, 1).shape, (5, 1))
    assert_equal(gaussian_random_matrix(5, 5).shape, (5, 5))
    assert_equal(gaussian_random_matrix(1, 1).shape, (1, 1))


def test_basic_property_of_random_matrix():
    """Check basic properties of random matrix generation"""
    for random_matrix in [gaussian_random_matrix,
                          bernouilli_random_matrix]:

        check_input_size_random_matrix(random_matrix)
        check_size_generated(random_matrix)


def test_gaussian_random_matrix():
    """Check some statical properties of gaussian random matrix"""
    # Check by a monte carlo estimator that each element of the random matrix
    # follow the distribution.
    #
    # Let's say that each element of a_{ij} of A is taken from
    #   a_ij ~ N(0.0, 1 / n_components).
    #
    n_components = 100
    n_features = 1000
    A = gaussian_random_matrix(n_components, n_features, random_state=0)

    assert_allclose(0.0, np.mean(A), atol=0.01)
    assert_allclose(np.var(A, ddof=1), 1 / n_components, rtol=1e-1)


def test_bernouilli_random_matrix():
    """Check some statical properties of (sparse) bernouilli random matrix"""
    # np.unique does not work on .todense sparse matrix
    n_components = 100
    n_features = 500

    for density in [0.3, 1.]:
        s = 1 / density
        A = np.array(bernouilli_random_matrix(n_components,
                                              n_features,
                                              density=density,
                                              random_state=0).todense())

        # Check possible values
        values = np.unique(A)
        assert_in(np.sqrt(s) / np.sqrt(n_components), values)
        assert_in(- np.sqrt(s) / np.sqrt(n_components), values)

        if density == 1.0:
            assert_equal(np.size(values), 2)
        else:
            assert_in(0., values)
            assert_equal(np.size(values), 3)

        # Check by a monte carlo estimator that each element of the
        # random matrix follow the proper distribution.
        #
        # Let's say that each element of a_{ij} of A is taken from
        #
        # - -sqrt(s) / sqrt(n_components)   with probability 1 / 2s
        # -  0                              with probability 1 - 1 / s
        # - +sqrt(s) / sqrt(n_components)   with probability 1 / 2s
        #
        assert_almost_equal(np.mean(A == 0.0),
                            1 - 1 / s, decimal=2)
        assert_almost_equal(np.mean(A == np.sqrt(s) / np.sqrt(n_components)),
                            1 / (2 * s), decimal=2)
        assert_almost_equal(np.mean(A == - np.sqrt(s) / np.sqrt(n_components)),
                            1 / (2 * s), decimal=2)

        assert_almost_equal(np.var(A == 0.0, ddof=1),
                            (1 - 1 / s) * 1 / s, decimal=2)
        assert_almost_equal(np.var(A == np.sqrt(s) / np.sqrt(n_components),
                                   ddof=1),
                            (1 - 1 / (2 * s)) * 1 / (2 * s), decimal=2)
        assert_almost_equal(np.var(A == - np.sqrt(s) / np.sqrt(n_components),
                                   ddof=1),<commit_msg>DOC: correct doc<commit_after>from __future__ import division
import warnings

import scipy.sparse as sp
import numpy as np
from numpy.testing import assert_allclose

from sklearn.metrics import euclidean_distances
from sklearn.random_projection import (johnson_lindenstrauss_min_dim,
                                       gaussian_random_matrix,
                                       bernouilli_random_matrix,
                                       BernouilliRandomProjection,
                                       GaussianRandomProjection,
                                       BaseRandomProjection,
                                       )

from sklearn.utils.testing import (assert_less,
                                   assert_raises, assert_raise_message,
                                   assert_array_equal,
                                   assert_equal, assert_almost_equal,
                                   assert_in,
                                   )


sparse_random_projection = [BernouilliRandomProjection]
dense_random_projection = [GaussianRandomProjection]
all_random_projection = set(sparse_random_projection + dense_random_projection)


# Make some random data with uniformly located non zero entries with
# gaussian distributed values
def make_sparse_random_data(n_samples, n_features, n_nonzeros):
    rng = np.random.RandomState(0)
    data_coo = sp.coo_matrix(
        (rng.randn(n_nonzeros),
        (rng.randint(n_samples, size=n_nonzeros),
         rng.randint(n_features, size=n_nonzeros))),
        shape=(n_samples, n_features))
    return data_coo.toarray(), data_coo.tocsr()


n_samples, n_features = (10, 1000)
n_nonzeros = n_samples * n_features / 100.
data, data_csr = make_sparse_random_data(n_samples, n_features, n_nonzeros)


###############################################################################
# test on JL lemma
###############################################################################
def test_invalid_jl_domain():
    assert_raises(ValueError, johnson_lindenstrauss_min_dim, 100, 1.1)
    assert_raises(ValueError, johnson_lindenstrauss_min_dim, 100, 0.0)
    assert_raises(ValueError, johnson_lindenstrauss_min_dim, 100, -0.1)


###############################################################################
# tests on random matrix generation
###############################################################################
def check_input_size_random_matrix(random_matrix):
    assert_raises(ValueError, random_matrix, 0, 0)
    assert_raises(ValueError, random_matrix, -1, 1)
    assert_raises(ValueError, random_matrix, 1, -1)
    assert_raises(ValueError, random_matrix, 1, 0)
    assert_raises(ValueError, random_matrix, -1, 0)


def check_size_generated(random_matrix):
    assert_equal(gaussian_random_matrix(1, 5).shape, (1, 5))
    assert_equal(gaussian_random_matrix(5, 1).shape, (5, 1))
    assert_equal(gaussian_random_matrix(5, 5).shape, (5, 5))
    assert_equal(gaussian_random_matrix(1, 1).shape, (1, 1))


def test_basic_property_of_random_matrix():
    """Check basic properties of random matrix generation"""
    for random_matrix in [gaussian_random_matrix,
                          bernouilli_random_matrix]:

        check_input_size_random_matrix(random_matrix)
        check_size_generated(random_matrix)


def test_gaussian_random_matrix():
    """Check some statical properties of gaussian random matrix"""
    # Check that the random matrix follow the proper distribution.
    # Let's say that each element of a_{ij} of A is taken from
    #   a_ij ~ N(0.0, 1 / n_components).
    #
    n_components = 100
    n_features = 1000
    A = gaussian_random_matrix(n_components, n_features, random_state=0)

    assert_allclose(0.0, np.mean(A), atol=0.01)
    assert_allclose(np.var(A, ddof=1), 1 / n_components, rtol=1e-1)


def test_bernouilli_random_matrix():
    """Check some statical properties of (sparse) bernouilli random matrix"""
    # np.unique does not work on .todense sparse matrix
    n_components = 100
    n_features = 500

    for density in [0.3, 1.]:
        s = 1 / density
        A = np.array(bernouilli_random_matrix(n_components,
                                              n_features,
                                              density=density,
                                              random_state=0).todense())

        # Check possible values
        values = np.unique(A)
        assert_in(np.sqrt(s) / np.sqrt(n_components), values)
        assert_in(- np.sqrt(s) / np.sqrt(n_components), values)

        if density == 1.0:
            assert_equal(np.size(values), 2)
        else:
            assert_in(0., values)
            assert_equal(np.size(values), 3)

        # Check that the random matrix follow the proper distribution.
        # Let's say that each element of a_{ij} of A is taken from
        #
        # - -sqrt(s) / sqrt(n_components)   with probability 1 / 2s
        # -  0                              with probability 1 - 1 / s
        # - +sqrt(s) / sqrt(n_components)   with probability 1 / 2s
        #
        assert_almost_equal(np.mean(A == 0.0),
                            1 - 1 / s, decimal=2)
        assert_almost_equal(np.mean(A == np.sqrt(s) / np.sqrt(n_components)),
                            1 / (2 * s), decimal=2)
        assert_almost_equal(np.mean(A == - np.sqrt(s) / np.sqrt(n_components)),
                            1 / (2 * s), decimal=2)

        assert_almost_equal(np.var(A == 0.0, ddof=1),
                            (1 - 1 / s) * 1 / s, decimal=2)
        assert_almost_equal(np.var(A == np.sqrt(s) / np.sqrt(n_components),
                                   ddof=1),
                            (1 - 1 / (2 * s)) * 1 / (2 * s), decimal=2)
        assert_almost_equal(np.var(A == - np.sqrt(s) / np.sqrt(n_components),
                                   ddof=1),
{'fraction_non_alphanumeric': 0.13941086846114778, 'fraction_numerical': 0.02031488065007618, 'mean_word_length': 2.2819444444444446, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3091891', 'n_tokens_mistral': 3684, 'n_tokens_neox': 3412, 'n_words': 746}