<commit_before>"""
The :mod:`sklearn.gp_search` includes utilities to fine-tune the parameters
of an estimator through a Gaussian Process model.
"""
from __future__ import print_function

# Author: <NAME> <http://bit.ly/SebastienDubois>,
# License: BSD 3 clause

import numpy as np
from scipy.stats import norm

from .gaussian_process import GaussianProcessRegressor
from .cross_validation import check_cv
from .cross_validation import _fit_and_score
from .metrics.scorer import check_scoring
from .base import is_classifier, clone
from .grid_search import BaseSearchCV


def _sample_candidates(n_candidates, param_bounds, param_is_int):
    n_parameters = param_is_int.shape[0]
    candidates = []

    for k in range(n_parameters):
        if param_is_int[k]:
            k_sample = np.asarray(
                np.random.rand(n_candidates)
                * np.float(param_bounds[k][1]-param_bounds[k][0])
                + param_bounds[k][0],
                dtype=np.int32)
        else:
            k_sample = np.asarray(
                np.random.rand(n_candidates)
                * np.float(param_bounds[k][1]-param_bounds[k][0])
                + param_bounds[k][0])
        candidates.append(k_sample)

    candidates = np.asarray(candidates).T
    return unique_rows(candidates)


def compute_expected_improvement(predictions, sigma, y_best):
    ei_array = np.zeros(predictions.shape[0])
    for i in range(ei_array.shape[0]):
        z = (y_best - predictions[i]) / sigma[i]
        ei_array[i] = sigma[i] * (z * norm.cdf(z) + norm.pdf(z))

    return ei_array


def unique_rows(ar):
    # keep only unique values in the ndarray ar
    # http://stackoverflow.com/questions/16970982/find-unique-rows-in-numpy-array

    b = np.ascontiguousarray(ar).view(
        np.dtype((np.void, ar.dtype.itemsize * ar.shape[1]))
        )
    _, idx = np.unique(b, return_index=True)
    idx = np.sort(idx)

    return ar[idx]


def is_in_ndarray(item, a):
    # look for element item in ndarray a
    # returns True if item is in a, and its index

    k = 0
    idx_val = np.asarray(range(a.shape[0]))
    idxk = range(a.shape[0])
    while(k < a.shape[1]):
        idxk = (a[idxk, k] == item[k])
        if np.sum(idxk > 0):
            k += 1
            idx_val = idx_val[idxk]
            idxk = list(idx_val)
        else:
            return False, 0

    return True, idx_val[0]


class GPSearchCV(BaseSearchCV):
    """
    Parameters
    ----------

    estimator : 1) sklearn estimator or 2) callable
        1 : object type that implements the "fit" and "predict" methods,
        as a classifier or a pipeline
        2 : a function that computes the output given a dictionnary of
        parameters. The returned value should be a list of one or more
        floats if score_format == 'cv', and a float if score_format ==
        'avg'

    parameters : dict, parameter space on which to optimize the estimator
        The keys of the dictionnary should be the names of the parameters,
        and the values should be lists of length 2; the first element being
        the type of the parameter ('int', 'float' or 'cat' [for categorical]),
        and the second element being a list of either the bounds between which
        to search (for 'int' and 'float') or the values the parameter can take
        (for 'cat')
        Example : parameters = {'kernel' :  ['cat', ['rbf', 'poly']],
                                 'd' : ['int', [1,3]],
                                 'C' : ['float',[1,10])}

    scoring : string, callable or None, optional
        A string (see sklearn's model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.
        Default is None.

    fit_params : dict, optional
        Parameters to pass to the fit method.

    cv : integer or cross-validation generator, optional
        Relevant if the estimator is an sklearn object.
        If an integer is passed, it is the number of folds.
        Specific cross-validation objects can be passed, see
        sklearn.cross_validation module for the list of possible objects
        Default is 5.

    acquisition function : string, optional
        Function to maximize in order to choose the next parameter to test.
        - Simple : maximize the predicted output
        - UCB : maximize the upper confidence bound
        - EI : maximizes the expected improvement
        Default is 'UCB'

    n_iter : int
        Total number of iterations to perform (including n_init and
        n_final_iter).
        Default is 100.

    n_init : int, optional
        Number of random iterations to perform before the smart search.
        Default is 30.

    n_final_iter : int, optional
        Number of final iterations, ie. smart iterations but with
        acquisition_function == 'Simple'
        Default is 5.

    n_candidates : int, optional
        Number of random candidates to sample for each GP iterations
        Default is 500.

    nugget : float, optional
        The nugget to set for the Gaussian Process.
        Default is 1.e-10.


    Attributes
    ----------

    best_params_ : dict, the parameter set, from those tested by the
        method _fit, that maximizes the mean of the cross-validation results.

    tested_parameters_ : ndarray, the parameters tested by _fit


    Examples
    -------
    >>> from sklearn.datasets import load_digits
    >>> iris = load_digits()
    >>> X, y = iris.data, iris.target
    >>> clf = RandomForestClassifier(n_estimators=20)
    >>> parameters = {"max_depth": ['int', [3, 3]],
                    "max_features": ['int', [1, 11]],
                    "min_samples_split": ['int', [1, 11]],
                    "min_samples_leaf": ['int', [1, 11]],
                    "bootstrap": ['cat', [True, False]],
                    "criterion": ['cat', ["gini", "entropy"]]}

    >>> search = GPSearchCV(parameters,
                            estimator=clf,
                            X=X,
                            y=y,
                            n_iter=20)
    >>> search._fit()

    """

    def __init__(self,
                 estimator,
                 parameters,
                 scoring=None,
                 fit_params=None,
                 cv=None,
                 acquisition_function='UCB',
                 refit=True,
                 n_init=10,
                 n_iter=20,
                 n_candidates=500,
                 gp_params={},
                 verbose=0):

        self.estimator = estimator
        self.parameters = parameters
        self.n_parameters = len(parameters)
        self.scoring = scoring
        self.acquisition_function = acquisition_function
        self.n_iter = n_iter
        self.n_init = n_init
        self.refit = refit
        self.n_candidates = n_candidates
        self.gp_params = gp_params
        self.param_names = list(parameters.keys())

        # FIXME: <commit_msg>FIX update API changes in docstring<commit_after>"""
The :mod:`sklearn.gp_search` includes utilities to fine-tune the parameters
of an estimator through a Gaussian Process model.
"""
from __future__ import print_function

# Author: <NAME> <http://bit.ly/SebastienDubois>,
# License: BSD 3 clause

import numpy as np
from scipy.stats import norm

from .gaussian_process import GaussianProcessRegressor
from .cross_validation import check_cv
from .cross_validation import _fit_and_score
from .metrics.scorer import check_scoring
from .base import is_classifier, clone
from .grid_search import BaseSearchCV


def _sample_candidates(n_candidates, param_bounds, param_is_int):
    n_parameters = param_is_int.shape[0]
    candidates = []

    for k in range(n_parameters):
        if param_is_int[k]:
            k_sample = np.asarray(
                np.random.rand(n_candidates)
                * np.float(param_bounds[k][1]-param_bounds[k][0])
                + param_bounds[k][0],
                dtype=np.int32)
        else:
            k_sample = np.asarray(
                np.random.rand(n_candidates)
                * np.float(param_bounds[k][1]-param_bounds[k][0])
                + param_bounds[k][0])
        candidates.append(k_sample)

    candidates = np.asarray(candidates).T
    return unique_rows(candidates)


def compute_expected_improvement(predictions, sigma, y_best):
    ei_array = np.zeros(predictions.shape[0])
    for i in range(ei_array.shape[0]):
        z = (y_best - predictions[i]) / sigma[i]
        ei_array[i] = sigma[i] * (z * norm.cdf(z) + norm.pdf(z))

    return ei_array


def unique_rows(ar):
    # keep only unique values in the ndarray ar
    # http://stackoverflow.com/questions/16970982/find-unique-rows-in-numpy-array

    b = np.ascontiguousarray(ar).view(
        np.dtype((np.void, ar.dtype.itemsize * ar.shape[1]))
        )
    _, idx = np.unique(b, return_index=True)
    idx = np.sort(idx)

    return ar[idx]


def is_in_ndarray(item, a):
    # look for element item in ndarray a
    # returns True if item is in a, and its index

    k = 0
    idx_val = np.asarray(range(a.shape[0]))
    idxk = range(a.shape[0])
    while(k < a.shape[1]):
        idxk = (a[idxk, k] == item[k])
        if np.sum(idxk > 0):
            k += 1
            idx_val = idx_val[idxk]
            idxk = list(idx_val)
        else:
            return False, 0

    return True, idx_val[0]


class GPSearchCV(BaseSearchCV):
    """
    Parameters
    ----------

    estimator : 1) sklearn estimator or 2) callable
        1 : object type that implements the "fit" and "predict" methods,
        as a classifier or a pipeline
        2 : a function that computes the output given a dictionnary of
        parameters. The returned value should be a list of one or more
        floats if score_format == 'cv', and a float if score_format ==
        'avg'

    parameters : dict, parameter space on which to optimize the estimator
        The keys of the dictionnary should be the names of the parameters,
        and the values should be lists of length 2; the first element being
        the type of the parameter ('int', 'float' or 'cat' [for categorical]),
        and the second element being a list of either the bounds between which
        to search (for 'int' and 'float') or the values the parameter can take
        (for 'cat')
        Example : parameters = {'kernel' :  ['cat', ['rbf', 'poly']],
                                 'd' : ['int', [1,3]],
                                 'C' : ['float',[1,10])}

    scoring : string, callable or None, optional
        A string (see sklearn's model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.
        Default is None.

    fit_params : dict, optional
        Parameters to pass to the fit method.

    cv : integer or cross-validation generator, optional
        Relevant if the estimator is an sklearn object.
        If an integer is passed, it is the number of folds.
        Specific cross-validation objects can be passed, see
        sklearn.cross_validation module for the list of possible objects
        Default is 5.

    acquisition function : string, optional
        Function to maximize in order to choose the next parameter to test.
        - Simple : maximize the predicted output
        - UCB : maximize the upper confidence bound
        - EI : maximizes the expected improvement
        Default is 'UCB'

    n_iter : int
        Total number of iterations to perform (including n_init and
        n_final_iter).
        Default is 100.

    n_init : int, optional
        Number of random iterations to perform before the smart search.
        Default is 30.

    n_final_iter : int, optional
        Number of final iterations, ie. smart iterations but with
        acquisition_function == 'Simple'
        Default is 5.

    n_candidates : int, optional
        Number of random candidates to sample for each GP iterations
        Default is 500.

    nugget : float, optional
        The nugget to set for the Gaussian Process.
        Default is 1.e-10.


    Attributes
    ----------

    best_params_ : dict, the parameter set, from those tested by the
        method _fit, that maximizes the mean of the cross-validation results.

    tested_parameters_ : ndarray, the parameters tested by _fit


    Examples
    -------
    >>> from sklearn.datasets import load_digits
    >>> iris = load_digits()
    >>> X, y = iris.data, iris.target
    >>> clf = RandomForestClassifier(n_estimators=20)
    >>> parameters = {"max_depth": ['int', [3, 3]],
                    "max_features": ['int', [1, 11]],
                    "min_samples_split": ['int', [1, 11]],
                    "min_samples_leaf": ['int', [1, 11]],
                    "bootstrap": ['cat', [True, False]],
                    "criterion": ['cat', ["gini", "entropy"]]}

    >>> search = GPSearchCV(parameters,
                            estimator=clf,
                            n_iter=20)
    >>> search.fit(X, y)

    """

    def __init__(self,
                 estimator,
                 parameters,
                 scoring=None,
                 fit_params=None,
                 cv=None,
                 acquisition_function='UCB',
                 refit=True,
                 n_init=10,
                 n_iter=20,
                 n_candidates=500,
                 gp_params={},
                 verbose=0):

        self.estimator = estimator
        self.parameters = parameters
        self.n_parameters = len(parameters)
        self.scoring = scoring
        self.acquisition_function = acquisition_function
        self.n_iter = n_iter
        self.n_init = n_init
        self.refit = refit
        self.n_candidates = n_candidates
        self.gp_params = gp_params
        self.param_names = list(parameters.keys())

        # FIXME: 
{'fraction_non_alphanumeric': 0.09502197248036885, 'fraction_numerical': 0.01080613788631943, 'mean_word_length': 2.0842035103310375, 'pattern_counts': {'":': 12, '<': 9, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3473383', 'n_tokens_mistral': 4148, 'n_tokens_neox': 3909, 'n_words': 1264}