<commit_before>__author__ = '<NAME>'

from apsis.optimizers.optimizer import Optimizer
from apsis.optimizers.random_search import RandomSearch
from apsis.models.parameter_definition import *
from apsis.utilities.randomization import check_random_state
from apsis.models.candidate import Candidate
from apsis.optimizers.bayesian.acquisition_functions import *
from apsis.utilities.import_utils import import_if_exists
import GPy
import logging

mcmc_imported, pm = import_if_exists("pymcmc")

class SimpleBayesianOptimizer(Optimizer):
    """
    This implements a simple bayesian optimizer.

    It is simple because it only implements the simplest form - no freeze-thaw,
    (currently) no multiple workers, only numeric parameters.

    Attributes
    ----------
    SUPPORTED_PARAM_TYPES : list of ParamDefs
        The supported parameter types. Currently only numberic and position.
    kernel : GPy Kernel
        The Kernel to be used with the gp.
    acquisition_function : acquisition_function
        The acquisition function to use
    acquisition_hyperparams :
        The acquisition hyperparameters.
    random_state : scipy random_state or int.
        The scipy random state or object to initialize one. For reproduction.
    random_searcher : RandomSearch
        The random search instance used to generate the first
        initial_random_runs candidates.
    gp : GPy gaussian process
        The gaussian process used here.
    initial_random_runs : int
        The number of initial random runs before using the GP. Default is 10.
    num_gp_restarts : int
        GPy's optimization requires restarts to find a good solution. This
        parameter controls this. Default is 10.
    logger: logger
        The logger instance for this object.
    """
    SUPPORTED_PARAM_TYPES = [NumericParamDef, PositionParamDef]

    kernel = None
    kernel_params = None
    acquisition_function = None
    acquisition_hyperparams = None

    random_state = None
    random_searcher = None

    gp = None
    mcmc = False
    initial_random_runs = 10
    num_gp_restarts = 10

    num_precomputed = None

    logger = None

    def __init__(self, optimizer_arguments=None):
        """
        Initializes a bayesian optimizer.

        Parameters
        ----------
        optimizer_arguments: dict of string keys
            Sets the possible arguments for this optimizer. Available are:
            "initial_random_runs" : int, optional
                The number of initial random runs before using the GP. Default
                is 10.
            "random_state" : scipy random state, optional
                The scipy random state or object to initialize one. Default is
                None.
            "acquisition_hyperparameters" : dict, optional
                dictionary of acquisition-function hyperparameters
            "num_gp_restarts" : int
                GPy's optimization requires restarts to find a good solution.
                This parameter controls this. Default is 10.
            "acquisition" : AcquisitionFunction
                The acquisition function to use. Default is
                ExpectedImprovement.
            "num_precomputed" : int
                The number of points that should be kept precomputed for faster
                multiple workers.
        """
        self.logger = get_logger(self)
        if optimizer_arguments is None:
            optimizer_arguments = {}
        self.initial_random_runs = optimizer_arguments.get(
            'initial_random_runs', self.initial_random_runs)
        self.random_state = check_random_state(
            optimizer_arguments.get('random_state', None))
        self.acquisition_hyperparams = optimizer_arguments.get(
            'acquisition_hyperparams', None)
        self.num_gp_restarts = optimizer_arguments.get(
            'num_gp_restarts', self.num_gp_restarts)
        if not isinstance(optimizer_arguments.get('acquisition'), AcquisitionFunction):
            self.acquisition_function = optimizer_arguments.get(
                'acquisition', ExpectedImprovement)(self.acquisition_hyperparams)
        else:
            self.acquisition_function = optimizer_arguments.get("acquisition")
        self.kernel_params = optimizer_arguments.get("kernel_params", {})
        self.kernel = optimizer_arguments.get("kernel", "matern52")
        self.random_searcher = RandomSearch({"random_state": self.random_state})

        if mcmc_imported:
            self.mcmc = optimizer_arguments.get("mcmc", False)
        else:
            self.mcmc = False

        self.num_precomputed = optimizer_arguments.get('num_precomputed', 10)
        self.logger.info("Bayesian optimization initialized.")

    def get_next_candidates(self, experiment, num_candidates=None):
        if not self._is_experiment_supported(experiment):
            self.logger.error("Experiment not supported. Supported parameter"
                              " types are: " + str(self.SUPPORTED_PARAM_TYPES))
            raise ValueError("Experiment not supported. Supported parameter"
                              " types are: " + str(self.SUPPORTED_PARAM_TYPES))
        if num_candidates is None:
            num_candidates = self.num_precomputed
        #check whether a random search is necessary.
        if len(experiment.candidates_finished) < self.initial_random_runs:
            return self.random_searcher.get_next_candidates(experiment, num_candidates)

        self._refit(experiment)
        #TODO refitted must be set, too.
        candidates = []
        new_candidate_points = self.acquisition_function.compute_proposals(
            self.gp, experiment, number_proposals=num_candidates)

        for point_and_value in new_candidate_points:
            #get the the candidate point which is the first entry in the tuple.
            point_candidate = Candidate(experiment.warp_pt_out(point_and_value[0]))
            candidates.append(point_candidate)
        return candidates



    def _refit(self, experiment):
        """
        Refits the GP with the data from experiment.

        Parameters
        ----------
        experiment : experiment
            The experiment on which to refit this gp.
        """
        candidate_matrix = np.zeros((len(experiment.candidates_finished),
                                     len(experiment.parameter_definitions)))
        results_vector = np.zeros((len(experiment.candidates_finished), 1))

        param_names = sorted(experiment.parameter_definitions.keys())
        self.kernel = self._check_kernel(self.kernel, len(param_names),
                                         kernel_params=self.kernel_params)
        for i, c in enumerate(experiment.candidates_finished):
            warped_in = experiment.warp_pt_in(c.params)
            param_values = []
            for pn in param_names:
                param_values.append(warped_in[pn])
            candidate_matrix[i, :] = param_values
            results_vector[i] = c.result
<commit_msg>#117. Changed bayesian optimization to work with new warping system.<commit_after>__author__ = '<NAME>'

from apsis.optimizers.optimizer import Optimizer
from apsis.optimizers.random_search import RandomSearch
from apsis.models.parameter_definition import *
from apsis.utilities.randomization import check_random_state
from apsis.models.candidate import Candidate
from apsis.optimizers.bayesian.acquisition_functions import *
from apsis.utilities.import_utils import import_if_exists
import GPy
import logging

mcmc_imported, pm = import_if_exists("pymcmc")

class SimpleBayesianOptimizer(Optimizer):
    """
    This implements a simple bayesian optimizer.

    It is simple because it only implements the simplest form - no freeze-thaw,
    (currently) no multiple workers, only numeric parameters.

    Attributes
    ----------
    SUPPORTED_PARAM_TYPES : list of ParamDefs
        The supported parameter types. Currently only numberic and position.
    kernel : GPy Kernel
        The Kernel to be used with the gp.
    acquisition_function : acquisition_function
        The acquisition function to use
    acquisition_hyperparams :
        The acquisition hyperparameters.
    random_state : scipy random_state or int.
        The scipy random state or object to initialize one. For reproduction.
    random_searcher : RandomSearch
        The random search instance used to generate the first
        initial_random_runs candidates.
    gp : GPy gaussian process
        The gaussian process used here.
    initial_random_runs : int
        The number of initial random runs before using the GP. Default is 10.
    num_gp_restarts : int
        GPy's optimization requires restarts to find a good solution. This
        parameter controls this. Default is 10.
    logger: logger
        The logger instance for this object.
    """
    SUPPORTED_PARAM_TYPES = [NumericParamDef, NominalParamDef]

    kernel = None
    kernel_params = None
    acquisition_function = None
    acquisition_hyperparams = None

    random_state = None
    random_searcher = None

    gp = None
    mcmc = False
    initial_random_runs = 10
    num_gp_restarts = 10

    num_precomputed = None

    logger = None

    def __init__(self, optimizer_arguments=None):
        """
        Initializes a bayesian optimizer.

        Parameters
        ----------
        optimizer_arguments: dict of string keys
            Sets the possible arguments for this optimizer. Available are:
            "initial_random_runs" : int, optional
                The number of initial random runs before using the GP. Default
                is 10.
            "random_state" : scipy random state, optional
                The scipy random state or object to initialize one. Default is
                None.
            "acquisition_hyperparameters" : dict, optional
                dictionary of acquisition-function hyperparameters
            "num_gp_restarts" : int
                GPy's optimization requires restarts to find a good solution.
                This parameter controls this. Default is 10.
            "acquisition" : AcquisitionFunction
                The acquisition function to use. Default is
                ExpectedImprovement.
            "num_precomputed" : int
                The number of points that should be kept precomputed for faster
                multiple workers.
        """
        self.logger = get_logger(self)
        if optimizer_arguments is None:
            optimizer_arguments = {}
        self.initial_random_runs = optimizer_arguments.get(
            'initial_random_runs', self.initial_random_runs)
        self.random_state = check_random_state(
            optimizer_arguments.get('random_state', None))
        self.acquisition_hyperparams = optimizer_arguments.get(
            'acquisition_hyperparams', None)
        self.num_gp_restarts = optimizer_arguments.get(
            'num_gp_restarts', self.num_gp_restarts)
        if not isinstance(optimizer_arguments.get('acquisition'), AcquisitionFunction):
            self.acquisition_function = optimizer_arguments.get(
                'acquisition', ExpectedImprovement)(self.acquisition_hyperparams)
        else:
            self.acquisition_function = optimizer_arguments.get("acquisition")
        self.kernel_params = optimizer_arguments.get("kernel_params", {})
        self.kernel = optimizer_arguments.get("kernel", "matern52")
        self.random_searcher = RandomSearch({"random_state": self.random_state})

        if mcmc_imported:
            self.mcmc = optimizer_arguments.get("mcmc", False)
        else:
            self.mcmc = False

        self.num_precomputed = optimizer_arguments.get('num_precomputed', 10)
        self.logger.info("Bayesian optimization initialized.")

    def get_next_candidates(self, experiment, num_candidates=None):
        if not self._is_experiment_supported(experiment):
            self.logger.error("Experiment not supported. Supported parameter"
                              " types are: " + str(self.SUPPORTED_PARAM_TYPES))
            raise ValueError("Experiment not supported. Supported parameter"
                              " types are: " + str(self.SUPPORTED_PARAM_TYPES))
        if num_candidates is None:
            num_candidates = self.num_precomputed
        #check whether a random search is necessary.
        if len(experiment.candidates_finished) < self.initial_random_runs:
            return self.random_searcher.get_next_candidates(experiment, num_candidates)

        self._refit(experiment)
        #TODO refitted must be set, too.
        candidates = []
        new_candidate_points = self.acquisition_function.compute_proposals(
            self.gp, experiment, number_proposals=num_candidates)

        for point_and_value in new_candidate_points:
            #get the the candidate point which is the first entry in the tuple.
            point_candidate = Candidate(experiment.warp_pt_out(point_and_value[0]))
            candidates.append(point_candidate)
        return candidates



    def _refit(self, experiment):
        """
        Refits the GP with the data from experiment.

        Parameters
        ----------
        experiment : experiment
            The experiment on which to refit this gp.
        """
        parameter_warped_size = 0
        for p in experiment.parameter_definitions.values():
            parameter_warped_size += p.warped_size()

        candidate_matrix = np.zeros((len(experiment.candidates_finished),
                                     parameter_warped_size))
        results_vector = np.zeros((len(experiment.candidates_finished), 1))

        param_names = sorted(experiment.parameter_definitions.keys())
        self.kernel = self._check_kernel(self.kernel, len(param_names),
                                         kernel_params=self.kernel_params)
        for i, c in enumerate(experiment.candidates_finished):
            warped_in = experiment.warp_pt_in(c.params)
            param_values = []
            for pn in param_names:
                param_values.extend(warped_in[pn])
            candidate_matrix[i, :] = param_values
            results_vector[i] = c.result

{'fraction_non_alphanumeric': 0.06613607550359206, 'fraction_numerical': 0.002817298211015636, 'mean_word_length': 2.4792942906150452, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7312042', 'n_tokens_mistral': 3835, 'n_tokens_neox': 3672, 'n_words': 1005}