<commit_before>import warnings

from nose.tools import assert_equal

import numpy as np
from numpy.testing import assert_array_almost_equal, assert_array_equal
from numpy.testing import assert_raises
from scipy.sparse import (bsr_matrix, coo_matrix, csc_matrix, csr_matrix,
                          dok_matrix, lil_matrix)
from scipy.spatial import cKDTree

from sklearn import neighbors, datasets

# load and shuffle iris dataset
iris = datasets.load_iris()
perm = np.random.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]

SPARSE_TYPES = (bsr_matrix, coo_matrix, csc_matrix, csr_matrix, dok_matrix,
                lil_matrix)
SPARSE_OR_DENSE = SPARSE_TYPES + (np.asarray,)

ALGORITHMS = ('ball_tree', 'brute', 'kd_tree', 'auto')
P = (1, 2, 3, 4, np.inf)


def test_warn_on_equidistant(n_samples=100, n_features=3, k=3):
    """test the production of a warning if equidistant points are discarded"""
    X = np.random.random(size=(n_samples, n_features))
    q = np.random.random(size=n_features)

    neigh = neighbors.NearestNeighbors(n_neighbors=k)
    neigh.fit(X[:-1])
    ind = neigh.kneighbors(q, return_distance=False)

    # make the last point identical to the furthest neighbor
    # querying this should set warning_flag to True
    X[-1] = X[ind[0, k - 1]]

    y = np.zeros(X.shape[0])

    expected_message = ("kneighbors: neighbor k+1 and neighbor k "
                        "have the same distance: results will be "
                        "dependent on data order.")

    algorithms = ('ball_tree', 'brute')
    estimators = (neighbors.KNeighborsClassifier,
                  neighbors.KNeighborsRegressor)

    for algorithm in algorithms:
        for estimator in estimators:
            with warnings.catch_warnings(record=True) as warn_queue:
                neigh = estimator(n_neighbors=k, algorithm=algorithm)
                neigh.fit(X, y)
                neigh.predict(q)
            print algorithm, estimator, len(warn_queue)

            assert_equal(len(warn_queue), 1)
            assert_equal(str(warn_queue[0].message), expected_message)


def test_unsupervised_kneighbors(n_samples=20, n_features=5,
                                 n_query_pts=2, n_neighbors=5,
                                 random_state=0):
    """Test unsupervised neighbors methods"""
    rng = np.random.RandomState(random_state)

    X = rng.rand(n_samples, n_features)

    test = rng.rand(n_query_pts, n_features)

    for p in P:
        results_nodist = []
        results = []<commit_msg>COSMIT: remove print<commit_after>import warnings

from nose.tools import assert_equal

import numpy as np
from numpy.testing import assert_array_almost_equal, assert_array_equal
from numpy.testing import assert_raises
from scipy.sparse import (bsr_matrix, coo_matrix, csc_matrix, csr_matrix,
                          dok_matrix, lil_matrix)
from scipy.spatial import cKDTree

from sklearn import neighbors, datasets

# load and shuffle iris dataset
iris = datasets.load_iris()
perm = np.random.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]

SPARSE_TYPES = (bsr_matrix, coo_matrix, csc_matrix, csr_matrix, dok_matrix,
                lil_matrix)
SPARSE_OR_DENSE = SPARSE_TYPES + (np.asarray,)

ALGORITHMS = ('ball_tree', 'brute', 'kd_tree', 'auto')
P = (1, 2, 3, 4, np.inf)


def test_warn_on_equidistant(n_samples=100, n_features=3, k=3):
    """test the production of a warning if equidistant points are discarded"""
    X = np.random.random(size=(n_samples, n_features))
    q = np.random.random(size=n_features)

    neigh = neighbors.NearestNeighbors(n_neighbors=k)
    neigh.fit(X[:-1])
    ind = neigh.kneighbors(q, return_distance=False)

    # make the last point identical to the furthest neighbor
    # querying this should set warning_flag to True
    X[-1] = X[ind[0, k - 1]]

    y = np.zeros(X.shape[0])

    expected_message = ("kneighbors: neighbor k+1 and neighbor k "
                        "have the same distance: results will be "
                        "dependent on data order.")

    algorithms = ('ball_tree', 'brute')
    estimators = (neighbors.KNeighborsClassifier,
                  neighbors.KNeighborsRegressor)

    for algorithm in algorithms:
        for estimator in estimators:
            with warnings.catch_warnings(record=True) as warn_queue:
                neigh = estimator(n_neighbors=k, algorithm=algorithm)
                neigh.fit(X, y)
                neigh.predict(q)

            assert_equal(len(warn_queue), 1)
            assert_equal(str(warn_queue[0].message), expected_message)


def test_unsupervised_kneighbors(n_samples=20, n_features=5,
                                 n_query_pts=2, n_neighbors=5,
                                 random_state=0):
    """Test unsupervised neighbors methods"""
    rng = np.random.RandomState(random_state)

    X = rng.rand(n_samples, n_features)

    test = rng.rand(n_query_pts, n_features)

    for p in P:
        results_nodist = []
        results = []
{'fraction_non_alphanumeric': 0.0963402571711177, 'fraction_numerical': 0.009099901088031652, 'mean_word_length': 2.9102861562258315, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6528434', 'n_tokens_mistral': 1712, 'n_tokens_neox': 1638, 'n_words': 384}