<commit_before>import json
import logging
import os
import time

import george
import numpy as np

from robo.acquisition_functions.ei import EI
from robo.acquisition_functions.information_gain_per_unit_cost import InformationGainPerUnitCost
from robo.acquisition_functions.marginalization import MarginalizationGPMCMC
from robo.initial_design import init_latin_hypercube_sampling
from robo.maximizers.random_sampling import RandomSampling
from robo.models.fabolas_gp import FabolasGPMCMC
from robo.priors.env_priors import EnvPrior
from robo.util.incumbent_estimation import projected_incumbent_estimation

logger = logging.getLogger(__name__)


def transform(s, s_min, s_max):
    s_transform = (np.log2(s) - np.log2(s_min)) / (np.log2(s_max) - np.log2(s_min))
    return s_transform


def retransform(s_transform, s_min, s_max):
    s = np.rint(2 ** (s_transform * (np.log2(s_max) - np.log2(s_min)) + np.log2(s_min)))
    return int(s)


def fabolas(objective_function, lower, upper, s_min, s_max,
            n_init=40, num_iterations=100, subsets=[256, 128, 64], inc_estimation="mean",
            burnin=100, chain_length=100, n_hypers=12, output_path=None, rng=None):
    """
    Fast Bayesian Optimization of Machine Learning Hyperparameters
    on Large Datasets

    Fast Bayesian Optimization of Machine Learning Hyperparameters on Large Datasets
    <NAME> and <NAME> and <NAME> and <NAME> and <NAME>
    http://arxiv.org/abs/1605.07079

    Parameters
    ----------
    objective_function: function
        Objective function that will be optimized
    lower: np.array(D,)
        Lower bound of the input space
    upper: np.array(D,)
        Upper bound of the input space
    s_min: int
        Minimum number of data points for the training data set
    s_max: int
        Maximum number of data points for the training data set
    n_init: int
        Number of initial design points
    n_hypers: int
        Number of hyperparameter samples for the GP
    subsets: list
        The ratio of the subsets size of the initial design.
        For example if subsets=[256, 128, 64] then the each point of the
        initial design is evaluated on s_max/256, s_max/128 and s_max/64
    num_iterations: int
        Number of iterations
    chain_length : int
        The length of the MCMC chain for each walker.
    burnin : int
        The number of burnin steps before the actual MCMC sampling starts.
    output_path: string
        Specifies the path where the intermediate output after each iteration will be saved.
        If None no output will be saved to disk.
    rng: numpy.random.RandomState
        Random number generator
    inc_estimation: string
        Specifies how the incumbent is estimated:
            "last_seen" : determined the incumbent as the configuration that achieved the highest observed value
            "mean" :  determined the incumbent as the configuration that achieved the highest predicted mean value
    Returns
    -------
        dict
    """

    assert n_init * len(
        subsets) <= num_iterations, "Number of initial design point (n_init * len(subsets)) " \
                                    "has to be <= than the number of iterations"
    assert lower.shape[0] == upper.shape[0], "Dimension miss match between upper and lower bound"

    time_start = time.time()
    if rng is None:
        rng = np.random.RandomState(np.random.randint(0, 10000))

    n_dims = lower.shape[0]

    # Bookkeeping
    time_func_eval = []
    time_overhead = []
    incumbents = []
    runtime = []

    X = []
    y = []
    c = []

    # Define model for the objective function
    cov_amp = 1  # Covariance amplitude
    kernel = cov_amp

    # ARD Kernel for the configuration space
    for d in range(n_dims):
        kernel *= george.kernels.Matern52Kernel(np.ones([1]) * 0.01,
                                                ndim=n_dims + 1, axes=d)

    # Kernel for the environmental variable
    # We use (1-s)**2 as basis function for the Bayesian linear kernel
    env_kernel = george.kernels.BayesianLinearRegressionKernel(log_a=0, log_b=0,
                                                               ndim=n_dims + 1,
                                                               axes=n_dims)
    kernel *= env_kernel

    # Take 3 times more samples than we have hyperparameters
    if n_hypers < 2 * len(kernel):
        n_hypers = 3 * len(kernel)
        if n_hypers % 2 == 1:
            n_hypers += 1

    prior = EnvPrior(len(kernel) + 1,
                     n_ls=n_dims,
                     n_lr=2,
                     rng=rng)

    quadratic_bf = lambda x: (1 - x) ** 2
    linear_bf = lambda x: x

    model_objective = FabolasGPMCMC(kernel,
                                    prior=prior,
                                    burnin_steps=burnin,
                                    chain_length=chain_length,
                                    n_hypers=n_hypers,
                                    normalize_output=False,
                                    basis_func=quadratic_bf,
                                    lower=lower,
                                    upper=upper,
                                    rng=rng)

    # Define model for the cost function
    cost_cov_amp = 1

    cost_kernel = cost_cov_amp

    # ARD Kernel for the configuration space
    for d in range(n_dims):
        cost_kernel *= george.kernels.Matern52Kernel(np.ones([1]) * 0.01,
                                                     ndim=n_dims + 1, axes=d)

    cost_env_kernel = george.kernels.BayesianLinearRegressionKernel(log_a=0, log_b=0,
                                                                    ndim=n_dims + 1,
                                                                    axes=n_dims)
    cost_kernel *= cost_env_kernel

    cost_prior = EnvPrior(len(cost_kernel) + 1,
                          n_ls=n_dims,
                          n_lr=2,
                          rng=rng)

    model_cost = FabolasGPMCMC(cost_kernel,
                               prior=cost_prior,
                               burnin_steps=burnin,
                               chain_length=chain_length,
                               n_hypers=n_hypers,
                               basis_func=linear_bf,
                               normalize_output=False,
                               lower=lower,
                               upper=upper,
                               rng=rng)

    # Extend input space by task variable
    extend_lower = np.append(lower, 0)
    extend_upper = np.append(upper, 1)
    is_env = np.zeros(extend_lower.shape[0])
    is_env[-1] = 1

    # Define acquisition function and maximizer
    ig = InformationGainPerUnitCost(model_objective,<commit_msg>change default<commit_after>import json
import logging
import os
import time

import george
import numpy as np

from robo.acquisition_functions.ei import EI
from robo.acquisition_functions.information_gain_per_unit_cost import InformationGainPerUnitCost
from robo.acquisition_functions.marginalization import MarginalizationGPMCMC
from robo.initial_design import init_latin_hypercube_sampling
from robo.maximizers.random_sampling import RandomSampling
from robo.models.fabolas_gp import FabolasGPMCMC
from robo.priors.env_priors import EnvPrior
from robo.util.incumbent_estimation import projected_incumbent_estimation

logger = logging.getLogger(__name__)


def transform(s, s_min, s_max):
    s_transform = (np.log2(s) - np.log2(s_min)) / (np.log2(s_max) - np.log2(s_min))
    return s_transform


def retransform(s_transform, s_min, s_max):
    s = np.rint(2 ** (s_transform * (np.log2(s_max) - np.log2(s_min)) + np.log2(s_min)))
    return int(s)


def fabolas(objective_function, lower, upper, s_min, s_max,
            n_init=40, num_iterations=100, subsets=[256, 128, 64], inc_estimation="mean",
            burnin=100, chain_length=100, n_hypers=12, output_path=None, rng=None):
    """
    Fast Bayesian Optimization of Machine Learning Hyperparameters
    on Large Datasets

    Fast Bayesian Optimization of Machine Learning Hyperparameters on Large Datasets
    <NAME> and <NAME> and <NAME> and <NAME> and <NAME>
    http://arxiv.org/abs/1605.07079

    Parameters
    ----------
    objective_function: function
        Objective function that will be optimized
    lower: np.array(D,)
        Lower bound of the input space
    upper: np.array(D,)
        Upper bound of the input space
    s_min: int
        Minimum number of data points for the training data set
    s_max: int
        Maximum number of data points for the training data set
    n_init: int
        Number of initial design points
    n_hypers: int
        Number of hyperparameter samples for the GP
    subsets: list
        The ratio of the subsets size of the initial design.
        For example if subsets=[256, 128, 64] then the each point of the
        initial design is evaluated on s_max/256, s_max/128 and s_max/64
    num_iterations: int
        Number of iterations
    chain_length : int
        The length of the MCMC chain for each walker.
    burnin : int
        The number of burnin steps before the actual MCMC sampling starts.
    output_path: string
        Specifies the path where the intermediate output after each iteration will be saved.
        If None no output will be saved to disk.
    rng: numpy.random.RandomState
        Random number generator
    inc_estimation: string
        Specifies how the incumbent is estimated:
            "last_seen" : determined the incumbent as the configuration that achieved the highest observed value
            "mean" :  determined the incumbent as the configuration that achieved the highest predicted mean value
    Returns
    -------
        dict
    """

    assert n_init * len(
        subsets) <= num_iterations, "Number of initial design point (n_init * len(subsets)) " \
                                    "has to be <= than the number of iterations"
    assert lower.shape[0] == upper.shape[0], "Dimension miss match between upper and lower bound"

    time_start = time.time()
    if rng is None:
        rng = np.random.RandomState(np.random.randint(0, 10000))

    n_dims = lower.shape[0]

    # Bookkeeping
    time_func_eval = []
    time_overhead = []
    incumbents = []
    runtime = []

    X = []
    y = []
    c = []

    # Define model for the objective function
    cov_amp = 1  # Covariance amplitude
    kernel = cov_amp

    # ARD Kernel for the configuration space
    for d in range(n_dims):
        kernel *= george.kernels.Matern52Kernel(np.ones([1]) * 0.01,
                                                ndim=n_dims + 1, axes=d)

    # Kernel for the environmental variable
    # We use (1-s)**2 as basis function for the Bayesian linear kernel
    env_kernel = george.kernels.BayesianLinearRegressionKernel(log_a=0.1, log_b=0.1,
                                                               ndim=n_dims + 1,
                                                               axes=n_dims)
    kernel *= env_kernel

    # Take 3 times more samples than we have hyperparameters
    if n_hypers < 2 * len(kernel):
        n_hypers = 3 * len(kernel)
        if n_hypers % 2 == 1:
            n_hypers += 1

    prior = EnvPrior(len(kernel) + 1,
                     n_ls=n_dims,
                     n_lr=2,
                     rng=rng)

    quadratic_bf = lambda x: (1 - x) ** 2
    linear_bf = lambda x: x

    model_objective = FabolasGPMCMC(kernel,
                                    prior=prior,
                                    burnin_steps=burnin,
                                    chain_length=chain_length,
                                    n_hypers=n_hypers,
                                    normalize_output=False,
                                    basis_func=quadratic_bf,
                                    lower=lower,
                                    upper=upper,
                                    rng=rng)

    # Define model for the cost function
    cost_cov_amp = 1

    cost_kernel = cost_cov_amp

    # ARD Kernel for the configuration space
    for d in range(n_dims):
        cost_kernel *= george.kernels.Matern52Kernel(np.ones([1]) * 0.01,
                                                     ndim=n_dims + 1, axes=d)

    cost_env_kernel = george.kernels.BayesianLinearRegressionKernel(log_a=0.1, log_b=0.1,
                                                                    ndim=n_dims + 1,
                                                                    axes=n_dims)
    cost_kernel *= cost_env_kernel

    cost_prior = EnvPrior(len(cost_kernel) + 1,
                          n_ls=n_dims,
                          n_lr=2,
                          rng=rng)

    model_cost = FabolasGPMCMC(cost_kernel,
                               prior=cost_prior,
                               burnin_steps=burnin,
                               chain_length=chain_length,
                               n_hypers=n_hypers,
                               basis_func=linear_bf,
                               normalize_output=False,
                               lower=lower,
                               upper=upper,
                               rng=rng)

    # Extend input space by task variable
    extend_lower = np.append(lower, 0)
    extend_upper = np.append(upper, 1)
    is_env = np.zeros(extend_lower.shape[0])
    is_env[-1] = 1

    # Define acquisition function and maximizer
    ig = InformationGainPerUnitCost(model_objective,
{'fraction_non_alphanumeric': 0.06930181012190617, 'fraction_numerical': 0.015663095677872184, 'mean_word_length': 1.7737704918032786, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1264574', 'n_tokens_mistral': 3968, 'n_tokens_neox': 3701, 'n_words': 1072}