<commit_before>import multiprocessing
import os
import time
import traceback

import numpy as np
import pandas as pd
import pynisher

# JTS TODO: notify aaron to clean up these nasty nested modules
from ConfigSpace.configuration_space import Configuration
from ConfigSpace.util import impute_inactive_values

from smac.smbo.smbo import SMBO, get_types
from smac.scenario.scenario import Scenario
from smac.tae.execute_ta_run import StatusType
from smac.runhistory.runhistory import RunHistory
from smac.runhistory.runhistory2epm import RunHistory2EPM4Cost, \
    RunHistory2EPM4EIPS
from smac.epm.uncorrelated_mo_rf_with_instances import \
    UncorrelatedMultiObjectiveRandomForestWithInstances
from smac.epm.rf_with_instances import RandomForestWithInstances
from smac.smbo.acquisition import EIPS

import autosklearn.metalearning
from autosklearn.constants import *
from autosklearn.metalearning.mismbo import suggest_via_metalearning
from autosklearn.data.abstract_data_manager import AbstractDataManager
from autosklearn.data.competition_data_manager import CompetitionDataManager
from autosklearn.evaluation import eval_with_limits
from autosklearn.util import get_logger
from autosklearn.util import Backend
from autosklearn.metalearning.metalearning.meta_base import MetaBase
from autosklearn.metalearning.metafeatures.metafeatures import \
    calculate_all_metafeatures_with_labels, calculate_all_metafeatures_encoded_labels

SENTINEL = 'uiaeo'

EXCLUDE_META_FEATURES_CLASSIFICATION = {
    'Landmark1NN',
    'LandmarkDecisionNodeLearner',
    'LandmarkDecisionTree',
    'LandmarkLDA',
    'LandmarkNaiveBayes',
    'PCAFractionOfComponentsFor95PercentVariance',
    'PCAKurtosisFirstPC',
    'PCASkewnessFirstPC',
    'PCA'
}

EXCLUDE_META_FEATURES_REGRESSION = {
    'Landmark1NN',
    'LandmarkDecisionNodeLearner',
    'LandmarkDecisionTree',
    'LandmarkLDA',
    'LandmarkNaiveBayes',
    'PCAFractionOfComponentsFor95PercentVariance',
    'PCAKurtosisFirstPC',
    'PCASkewnessFirstPC',
    'NumberOfClasses',
    'ClassOccurences',
    'ClassProbabilityMin',
    'ClassProbabilityMax',
    'ClassProbabilityMean',
    'ClassProbabilitySTD',
    'ClassEntropy',
    'LandmarkRandomNodeLearner',
    'PCA',
}


# dataset helpers
def load_data(dataset_info, outputdir, tmp_dir=None, max_mem=None):
    if tmp_dir is None:
        tmp_dir = outputdir
    backend = Backend(outputdir, tmp_dir)
    try:
        D = backend.load_datamanager()
    except IOError:
        D = None

    # Datamanager probably doesn't exist
    if D is None:
        if max_mem is None:
            D = CompetitionDataManager(dataset_info, encode_labels=True)
        else:
            D = CompetitionDataManager(dataset_info, encode_labels=True, max_memory_in_mb=max_mem)
    return D


# metalearning helpers
def _calculate_metafeatures(data_feat_type, data_info_task, basename,
                            x_train, y_train, watcher, logger):
    # == Calculate metafeatures
    task_name = 'CalculateMetafeatures'
    watcher.start_task(task_name)
    categorical = [True if feat_type.lower() in ['categorical'] else False
                   for feat_type in data_feat_type]

    EXCLUDE_META_FEATURES = EXCLUDE_META_FEATURES_CLASSIFICATION \
        if data_info_task in CLASSIFICATION_TASKS else EXCLUDE_META_FEATURES_REGRESSION

    if data_info_task in [MULTICLASS_CLASSIFICATION, BINARY_CLASSIFICATION,
                          MULTILABEL_CLASSIFICATION, REGRESSION]:
        logger.info('Start calculating metafeatures for %s', basename)
        result = calculate_all_metafeatures_with_labels(
            x_train, y_train, categorical=categorical,
            dataset_name=basename+SENTINEL,
            dont_calculate=EXCLUDE_META_FEATURES, )
        for key in list(result.metafeature_values.keys()):
            if result.metafeature_values[key].type_ != 'METAFEATURE':
                del result.metafeature_values[key]

    else:
        result = None
        logger.info('Metafeatures not calculated')
    watcher.stop_task(task_name)
    logger.info(
        'Calculating Metafeatures (categorical attributes) took %5.2f',
        watcher.wall_elapsed(task_name))
    return result

def _calculate_metafeatures_encoded(basename, x_train, y_train, watcher,
                                    task, logger):
    EXCLUDE_META_FEATURES = EXCLUDE_META_FEATURES_CLASSIFICATION \
        if task in CLASSIFICATION_TASKS else EXCLUDE_META_FEATURES_REGRESSION

    task_name = 'CalculateMetafeaturesEncoded'
    watcher.start_task(task_name)
    result = calculate_all_metafeatures_encoded_labels(
        x_train, y_train, categorical=[False] * x_train.shape[1],
        dataset_name=basename+SENTINEL, dont_calculate=EXCLUDE_META_FEATURES)
    for key in list(result.metafeature_values.keys()):
        if result.metafeature_values[key].type_ != 'METAFEATURE':
            del result.metafeature_values[key]
    watcher.stop_task(task_name)
    logger.info(
        'Calculating Metafeatures (encoded attributes) took %5.2fsec',
        watcher.wall_elapsed(task_name))
    return result

def _get_metalearning_configurations(meta_base, basename, metric,
                                     configuration_space,
                                     task,
                                     initial_configurations_via_metalearning,
                                     is_sparse,
                                     watcher, logger):
    task_name = 'InitialConfigurations'
    watcher.start_task(task_name)
    try:
        metalearning_configurations = suggest_via_metalearning(
            meta_base, basename, metric,
            task,
            is_sparse == 1,
            initial_configurations_via_metalearning
        )
    except Exception as e:
        logger.error("Error getting metalearning configurations!")
        logger.error(str(e))
        logger.error(traceback.format_exc())
        metalearning_configurations = []
    watcher.stop_task(task_name)
    return metalearning_configurations

def _print_debug_info_of_init_configuration(initial_configurations, basename,
                                            time_for_task, logger, watcher):
    logger.debug('Initial Configurations: (%d)' % len(initial_configurations))
    for initial_configuration in initial_configurations:
        logger.debug(initial_configuration)
    logger.debug('Looking for initial configurations took %5.2fsec',
                 watcher.wall_elapsed('InitialConfigurations'))
    logger.info(
        'Time left for %s after finding initial configurations: %5.2fsec',
        basename, time_for_task - watcher.wall_elapsed(basename))


class AutoMLScenario(Scenario):
    """
    We specialize the smac3 scenario here as we would like
    to create it in code, without actually reading a smac scenario file
    """

    def __init__(self, config_space, limit, cutoff_time, metafeatures):
        self.logger = get_logger(self.__class__.__name__)

        # Give SMAC at least 5 seconds
        soft_limit = max(5, cutoff_time - 35)

        scenario_dict = {'cs': config_space,
                         'run_obj': 'quality',
                         'cutoff': soft_limit,
                         'algo_runs_timelimit': soft_limit,
                         'wallclock-limit': limit,
                         'features': metafeatures,
                         'instances': [[name] for name in metafeatures]}

        super(AutoMLScenario, self).__init__(scenario_dict)

class AutoMLSMBO(multiprocessing.Process):

    def __init__(self, config_space, dataset_name,
                 output_dir, tmp_dir,
                 total_walltime_limit,
                 func_eval_time_limit,
                 memory_limit,
                 watcher, start_num_run=1,
                 data_memory_limit=None,
                 num_metalearning_cfgs=25,
                 config_file=None,
                 smac_iters=1000,
                 seed=1,
                 metadata_directory=None,
                 resampling_strategy='holdout',
                 resampling_strategy_args=None,
                 acquisition_function='EI'):
        super(AutoMLSMBO, self).__init__()
        # data related
        self.dataset_name = dataset_name
        self.output_dir = output_dir
        self.tmp_dir = tmp_dir
        self.datamanager = None
        self.metric = None
        self.task = None

        # the configuration space
        self.config_space = config_space

        # Evaluation
        self.resampling_strategy = resampling_strategy
        if resampling_strategy_args is None:
            resampling_strategy_args = {}
        self.resampling_strategy_args = resampling_strategy_args

        # and a bunch of useful limits
        self.total_walltime_limit = int(total_walltime_limit)
        self.func_eval_time_limit = int(func_eval_time_limit)
        self.memory_limit = memory_limit
        self.data_memory_limit = data_memory_limit
        self.watcher = watcher
        self.num_metalearning_cfgs = num_metalearning_cfgs
        self.config_file = config_file
        self.seed = seed
        self.metadata_directory = metadata_directory
        self.smac_iters = smac_iters
        self.start_num_run = start_num_run
        self.acquisition_function = acquisition_function

        self.config_space.seed(self.seed)
        logger_name = self.__class__.__name__ + \
                      (":" + dataset_name if dataset_name is not None else "")
        self.logger = get_logger(logger_name)

    def reset_data_manager(self, max_mem=None):
        if max_mem is None:
            max_mem = self.data_memory_limit
        if self.datamanager is not None:
            del self.datamanager
        if isinstance(self.dataset_name, AbstractDataManager):
            self.datamanager = self.dataset_name
        else:
            self.datamanager = load_data(self.dataset_name,
                                         self.output_dir,
                                         self.tmp_dir,
                                         max_mem = max_mem)
        self.metric = self.datamanager.info['metric']
        self.task = self.datamanager.info['task']
        
    def collect_additional_subset_defaults(self):
        default_configs = []
        # == set default configurations
        # first enqueue the default configuration from our config space
        if self.datamanager.info["task"] in CLASSIFICATION_TASKS:
            config_dict = {'balancing:strategy': 'weighting',
                           'classifier:__choice__': 'sgd',
                           'classifier:sgd:loss': 'hinge',
                           'classifier:sgd:penalty': 'l2',
                           'classifier:sgd:alpha': 0.0001,
                           'classifier:sgd:fit_intercept': 'True',
                           'classifier:sgd:n_iter': 5,
                           'classifier:sgd:learning_rate': 'optimal',
                           'classifier:sgd:eta0': 0.01,
                           'classifier:sgd:average': 'True',
                           'imputation:strategy': 'mean',
                           'one_hot_encoding:use_minimum_fraction': 'True',
                           'one_hot_encoding:minimum_fraction': 0.1,
                           'preprocessor:__choice__': 'no_preprocessing',
                           'rescaling:__choice__': 'min/max'}
            try:
                config = Configuration(self.config_space, config_dict)
                default_configs.append(config)
            except ValueError as e:
                self.logger.warning("Second default configurations %s cannot"
                                    " be evaluated because of %s" %
                                    (config_dict, e))

            if self.datamanager.info["is_sparse"]:
                config_dict = {'classifier:__choice__': 'extra_trees',
                               'classifier:extra_trees:bootstrap': 'False',
                               'classifier:extra_trees:criterion': 'gini',
                               'classifier:extra_trees:max_depth': 'None',
                               'classifier:extra_trees:max_features': 1.0,
                               'classifier:extra_trees:min_samples_leaf': 5,
                               'classifier:extra_trees:min_samples_split': 5,
                               'classifier:extra_trees:min_weight_fraction_leaf': 0.0,
                               'classifier:extra_trees:n_estimators': 100,
                               'balancing:strategy': 'weighting',
                               'imputation:strategy': 'mean',
                               'one_hot_encoding:use_minimum_fraction': 'True',
                               'one_hot_encoding:minimum_fraction': 0.1,
                               'preprocessor:__choice__': 'truncatedSVD',
                               'preprocessor:truncatedSVD:target_dim': 20,
                               'rescaling:__choice__': 'min/max'}
            else:
                n_data_points = self.datamanager.data['X_train'].shape[0]
                percentile = 20. / n_data_points
                percentile = max(percentile, 2.)

                config_dict = {'classifier:__choice__': 'extra_trees',
                               'classifier:extra_trees:bootstrap': 'False',
                               'classifier:extra_trees:criterion': 'gini',
                               'classifier:extra_trees:max_depth': 'None',
                               'classifier:extra_trees:max_features': 1.0,
                               'classifier:extra_trees:min_samples_leaf': 5,
                               'classifier:extra_trees:min_samples_split': 5,
                               'classifier:extra_trees:min_weight_fraction_leaf': 0.0,
                               'classifier:extra_trees:n_estimators': 100,
                               'balancing:strategy': 'weighting',
                               'imputation:strategy': 'mean',
                               'one_hot_encoding:use_minimum_fraction': 'True',
                               'one_hot_encoding:minimum_fraction': 0.1,
                               'preprocessor:__choice__': 'select_percentile_classification',
                               'preprocessor:select_percentile_classification:percentile': percentile,
                               'preprocessor:select_percentile_classification:score_func': 'chi2',
                               'rescaling:__choice__': 'min/max'}

            try:
                config = Configuration(self.config_space, config_dict)
                default_configs.append(config)
            except ValueError as e:
                self.logger.warning("Third default configurations %s cannot"
                                    " be evaluated because of %s" %
                                    (config_dict, e))

            if self.datamanager.info["is_sparse"]:
                config_dict = {'balancing:strategy': 'weighting',
                               'classifier:__choice__': 'multinomial_nb',
                               'classifier:multinomial_nb:alpha': 1.0,
                               'classifier:multinomial_nb:fit_prior': 'True',
                               'imputation:strategy': 'mean',
                               'one_hot_encoding:use_minimum_fraction': 'True',
                               'one_hot_encoding:minimum_fraction': 0.1,
                               'preprocessor:__choice__': 'no_preprocessing',
                               'rescaling:__choice__': 'none'}
            else:
                config_dict = {'balancing:strategy': 'weighting',
                               'classifier:__choice__': 'gaussian_nb',
                               'imputation:strategy': 'mean',
                               'one_hot_encoding:use_minimum_fraction': 'True',
                               'one_hot_encoding:minimum_fraction': 0.1,
                               'preprocessor:__choice__': 'no_preprocessing',
                               'rescaling:__choice__': 'standardize'}
            try:
                config = Configuration(self.config_space, config_dict)
                default_configs.append(config)
            except ValueError as e:
                self.logger.warning("Forth default configurations %s cannot"
                                    " be evaluated because of %s" %
                                    (config_dict, e))

        elif self.datamanager.info["task"] in REGRESSION_TASKS:
            config_dict = {'regressor:__choice__': 'sgd',
                           'regressor:sgd:loss': 'squared_loss',
                           'regressor:sgd:penalty': 'l2',
                           'regressor:sgd:alpha': 0.0001,
                           'regressor:sgd:fit_intercept': 'True',
                           'regressor:sgd:n_iter': 5,
                           'regressor:sgd:learning_rate': 'optimal',
                           'regressor:sgd:eta0': 0.01,
                           'regressor:sgd:average': 'True',
                           'imputation:strategy': 'mean',
                           'one_hot_encoding:use_minimum_fraction': 'True',
                           'one_hot_encoding:minimum_fraction': 0.1,
                           'preprocessor:__choice__': 'no_preprocessing',
                           'rescaling:__choice__': 'min/max'}
            try:
                config = Configuration(self.config_space, config_dict)
                default_configs.append(config)
            except ValueError as e:
                self.logger.warning("Second default configurations %s cannot"
                                    " be evaluated because of %s" %
                                    (config_dict, e))

            if self.datamanager.info["is_sparse"]:
                config_dict = {'regressor:__choice__': 'extra_trees',
                               'regressor:extra_trees:bootstrap': 'False',
                               'regressor:extra_trees:criterion': 'mse',
                               'regressor:extra_trees:max_depth': 'None',
                               'regressor:extra_trees:max_features': 1.0,
                               'regressor:extra_trees:min_samples_leaf': 5,
                               'regressor:extra_trees:min_samples_split': 5,
                               'regressor:extra_trees:n_estimators': 100,
                               'imputation:strategy': 'mean',
                               'one_hot_encoding:use_minimum_fraction': 'True',
                               'one_hot_encoding:minimum_fraction': 0.1,
                               'preprocessor:__choice__': 'truncatedSVD',
                               'preprocessor:truncatedSVD:target_dim': 10,
                               'rescaling:__choice__': 'min/max'}
            else:
                config_dict = {'regressor:__choice__': 'extra_trees',
                               'regressor:extra_trees:bootstrap': 'False',
                               'regressor:extra_trees:criterion': 'mse',
                               'regressor:extra_trees:max_depth': 'None',
                               'regressor:extra_trees:max_features': 1.0,
                               'regressor:extra_trees:min_samples_leaf': 5,
                               'regressor:extra_trees:min_samples_split': 5,
                               'regressor:extra_trees:n_estimators': 100,
                               'imputation:strategy': 'mean',
                               'one_hot_encoding:use_minimum_fraction': 'True',
                               'one_hot_encoding:minimum_fraction': 0.1,
                               'preprocessor:__choice__': 'pca',
                               'preprocessor:pca:keep_variance': 0.9,
                               'preprocessor:pca:whiten': 'False',
                               'rescaling:__choice__': 'min/max'}

            try:
                config = Configuration(self.config_space, config_dict)
                default_configs.append(config)
            except ValueError as e:
                self.logger.warning("Third default configurations %s cannot"
                                    " be evaluated because of %s" %
                                    (config_dict, e))

        else:
            self.logger.info("Tasktype unknown: %s" %
                             TASK_TYPES_TO_STRING[self.datamanager.info[
                                 "task"]])

        return default_configs

    def collect_metalearning_suggestions(self, meta_base):
        metalearning_configurations = _get_metalearning_configurations(
            meta_base=meta_base,
            basename=self.dataset_name+SENTINEL,
            metric=self.metric,
            configuration_space=self.config_space,
            task=self.task,
            is_sparse=self.datamanager.info['is_sparse'],
            initial_configurations_via_metalearning=self.num_metalearning_cfgs,
            watcher=self.watcher,
            logger=self.logger)
        _print_debug_info_of_init_configuration(
            metalearning_configurations,
            self.dataset_name,
            self.total_walltime_limit,
            self.logger,
            self.watcher)

        return metalearning_configurations

    def _calculate_metafeatures(self):
        meta_features = _calculate_metafeatures(
            data_feat_type=self.datamanager.feat_type,
            data_info_task=self.datamanager.info['task'],
            x_train=self.datamanager.data['X_train'],
            y_train=self.datamanager.data['Y_train'],
            basename=self.dataset_name,
            watcher=self.watcher,
            logger=self.logger)
        return meta_features

    def _calculate_metafeatures_with_limits(self, time_limit):
        res = {}
        try:
            safe_mf = pynisher.enforce_limits(mem_in_mb=self.memory_limit,
                                              wall_time_in_s=int(time_limit),
                                              grace_period_in_s=30)(
                self._calculate_metafeatures)
            res = safe_mf()
        except Exception as e:
            self.logger.error('Error getting metafeatures: %s', str(e))

        return res

    def _calculate_metafeatures_encoded(self):
        meta_features_encoded = _calculate_metafeatures_encoded(
            self.dataset_name,
            self.datamanager.data['X_train'],
            self.datamanager.data['Y_train'],
            self.watcher,
            self.datamanager.info['task'],
            self.logger)
        return meta_features_encoded

    def _calculate_metafeatures_encoded_with_limits(self, time_limit):
        res = {}
        try:
            safe_mf = pynisher.enforce_limits(mem_in_mb=self.memory_limit,
                                              wall_time_in_s=int(time_limit),
                                              grace_period_in_s=30)(
                self._calculate_metafeatures_encoded)
            res = safe_mf()
        except Exception as e:
            self.logger.error('Error getting metafeatures (encoded) : %s',
                              str(e))

        return res
    
    def run(self):
        # we use pynisher here to enforce limits
        safe_smbo = pynisher.enforce_limits(mem_in_mb=self.memory_limit,
                                            wall_time_in_s=int(self.total_walltime_limit),
                                            grace_period_in_s=5)(self.run_smbo)
        safe_smbo(max_iters = self.smac_iters)
        

    def run_smbo(self, max_iters=1000):
        global evaluator

        # == first things first: load the datamanager
        self.reset_data_manager()
        
        # == Initialize SMBO stuff
        # first create a scenario
        seed = self.seed # TODO
        num_params = len(self.config_space.get_hyperparameters())
        # allocate a run history
        run_history = RunHistory()
        meta_runhistory = RunHistory()
        meta_runs_dataset_indices = {}
        num_run = self.start_num_run
        instance_id = self.dataset_name + SENTINEL

        # == Train on subset
        #    before doing anything, let us run the default_cfg
        #    on a subset of the available data to ensure that
        #    we at least have some models
        #    we will try three different ratios of decreasing magnitude
        #    in the hope that at least on the last one we will be able
        #    to get a model
        n_data = self.datamanager.data['X_train'].shape[0]
        subset_ratio = 10000. / n_data
        if subset_ratio >= 0.5:
            subset_ratio = 0.33
            subset_ratios = [subset_ratio, subset_ratio * 0.10]
        else:
            subset_ratios = [subset_ratio, 500. / n_data]
        self.logger.info("Training default configurations on a subset of "
                         "%d/%d data points." %
                         (int(n_data * subset_ratio), n_data))

        # the time limit for these function evaluations is rigorously
        # set to only 1/2 of a full function evaluation
        subset_time_limit = max(5, int(self.func_eval_time_limit / 2))
        # the configs we want to run on the data subset are:
        # 1) the default configs
        # 2) a set of configs we selected for training on a subset
        subset_configs = [self.config_space.get_default_configuration()] \
                          + self.collect_additional_subset_defaults()
        subset_config_succesful = [False] * len(subset_configs)
        for subset_config_id, next_config in enumerate(subset_configs):
            for i, ratio in enumerate(subset_ratios):
                self.reset_data_manager()
                n_data_subsample = int(n_data * ratio)

                # run the config, but throw away the result afterwards
                # since this cfg was evaluated only on a subset
                # and we don't want  to confuse SMAC
                self.logger.info("Starting to evaluate %d on SUBSET "
                                 "with size %d and time limit %ds.",
                                 num_run, n_data_subsample,
                                 subset_time_limit)
                self.logger.info(next_config)
                _info = eval_with_limits(
                    self.datamanager, self.tmp_dir, next_config,
                    seed, num_run,
                    self.resampling_strategy,
                    self.resampling_strategy_args,
                    self.memory_limit,
                    subset_time_limit, n_data_subsample)
                (duration, result, _, additional_run_info, status) = _info
                self.logger.info("Finished evaluating %d. configuration on SUBSET. "
                                 "Duration %f; loss %f; status %s; additional run "
                                 "info: %s ", num_run, duration, result,
                                 str(status), additional_run_info)

                num_run += 1
                if i < len(subset_ratios) - 1:
                    if status != StatusType.SUCCESS:
                        # Do not increase num_run here, because we will try
                        # the same configuration with less data
                        self.logger.info("A CONFIG did not finish "
                                         " for subset ratio %f -> going smaller",
                                         ratio)
                        continue
                    else:
                        self.logger.info("Finished SUBSET training sucessfully "
                                         "with ratio %f", ratio)
                        subset_config_succesful[subset_config_id] = True
                        break
                else:
                    if status != StatusType.SUCCESS:
                        self.logger.info("A CONFIG did not finish "
                                         " for subset ratio %f.",
                                         ratio)
                        continue
                    else:
                        self.logger.info("Finished SUBSET training sucessfully "
                                         "with ratio %f", ratio)
                        subset_config_succesful[subset_config_id] = True
                        break

        # Use the first non-failing configuration from the subsets as the new
        #  default configuration -> this guards us against the random forest
        # failing on large, sparse datasets
        default_cfg = None
        for subset_config_id, next_config in enumerate(subset_configs):
            if subset_config_succesful[subset_config_id]:
                default_cfg = next_config
                break
        if default_cfg is None:
            default_cfg = self.config_space.get_default_configuration()

        # == METALEARNING suggestions
        # we start by evaluating the defaults on the full dataset again
        # and add the suggestions from metalearning behind it

        if self.metadata_directory is None:
            metalearning_directory = os.path.dirname(
                autosklearn.metalearning.__file__)
            # There is no multilabel data in OpenML
            if self.task == MULTILABEL_CLASSIFICATION:
                meta_task = BINARY_CLASSIFICATION
            else:
                meta_task = self.task
            metadata_directory = os.path.join(
                metalearning_directory, 'files',
                '%s_%s_%s' % (METRIC_TO_STRING[self.metric],
                              TASK_TYPES_TO_STRING[meta_task],
                              'sparse' if self.datamanager.info['is_sparse']
                              else 'dense'))
            self.metadata_directory = metadata_directory

        self.logger.info('Metadata directory: %s', self.metadata_directory)
        meta_base = MetaBase(self.config_space, self.metadata_directory)

        metafeature_calculation_time_limit = int(
            self.total_walltime_limit / 4)
        metafeature_calculation_start_time = time.time()
        meta_features = self._calculate_metafeatures_with_limits(
            metafeature_calculation_time_limit)
        metafeature_calculation_end_time = time.time()
        metafeature_calculation_time_limit = \
            metafeature_calculation_time_limit - (
            metafeature_calculation_end_time -
            metafeature_calculation_start_time)

        if metafeature_calculation_time_limit <= 0.1:
            self.logger.warning('Time limit for metafeature calculation less '
                                'than 0.1 seconds (%f). Skipping calculation '
                                'of encoded metafeatures.')
            meta_features_encoded = None
        else:
            self.datamanager.perform1HotEncoding()
            meta_features_encoded = \
                self._calculate_metafeatures_encoded_with_limits(
                    metafeature_calculation_time_limit)

        # In case there is a problem calculating the encoded meta-features
        if meta_features is None:
            if meta_features_encoded is not None:
                meta_features = meta_features_encoded<commit_msg>FIX metafeature calculation<commit_after>import multiprocessing
import os
import time
import traceback

import numpy as np
import pandas as pd
import pynisher

# JTS TODO: notify aaron to clean up these nasty nested modules
from ConfigSpace.configuration_space import Configuration
from ConfigSpace.util import impute_inactive_values

from smac.smbo.smbo import SMBO, get_types
from smac.scenario.scenario import Scenario
from smac.tae.execute_ta_run import StatusType
from smac.runhistory.runhistory import RunHistory
from smac.runhistory.runhistory2epm import RunHistory2EPM4Cost, \
    RunHistory2EPM4EIPS
from smac.epm.uncorrelated_mo_rf_with_instances import \
    UncorrelatedMultiObjectiveRandomForestWithInstances
from smac.epm.rf_with_instances import RandomForestWithInstances
from smac.smbo.acquisition import EIPS

import autosklearn.metalearning
from autosklearn.constants import *
from autosklearn.metalearning.mismbo import suggest_via_metalearning
from autosklearn.data.abstract_data_manager import AbstractDataManager
from autosklearn.data.competition_data_manager import CompetitionDataManager
from autosklearn.evaluation import eval_with_limits
from autosklearn.util import get_logger
from autosklearn.util import Backend
from autosklearn.metalearning.metalearning.meta_base import MetaBase
from autosklearn.metalearning.metafeatures.metafeatures import \
    calculate_all_metafeatures_with_labels, calculate_all_metafeatures_encoded_labels

SENTINEL = 'uiaeo'

EXCLUDE_META_FEATURES_CLASSIFICATION = {
    'Landmark1NN',
    'LandmarkDecisionNodeLearner',
    'LandmarkDecisionTree',
    'LandmarkLDA',
    'LandmarkNaiveBayes',
    'PCAFractionOfComponentsFor95PercentVariance',
    'PCAKurtosisFirstPC',
    'PCASkewnessFirstPC',
    'PCA'
}

EXCLUDE_META_FEATURES_REGRESSION = {
    'Landmark1NN',
    'LandmarkDecisionNodeLearner',
    'LandmarkDecisionTree',
    'LandmarkLDA',
    'LandmarkNaiveBayes',
    'PCAFractionOfComponentsFor95PercentVariance',
    'PCAKurtosisFirstPC',
    'PCASkewnessFirstPC',
    'NumberOfClasses',
    'ClassOccurences',
    'ClassProbabilityMin',
    'ClassProbabilityMax',
    'ClassProbabilityMean',
    'ClassProbabilitySTD',
    'ClassEntropy',
    'LandmarkRandomNodeLearner',
    'PCA',
}


# dataset helpers
def load_data(dataset_info, outputdir, tmp_dir=None, max_mem=None):
    if tmp_dir is None:
        tmp_dir = outputdir
    backend = Backend(outputdir, tmp_dir)
    try:
        D = backend.load_datamanager()
    except IOError:
        D = None

    # Datamanager probably doesn't exist
    if D is None:
        if max_mem is None:
            D = CompetitionDataManager(dataset_info, encode_labels=True)
        else:
            D = CompetitionDataManager(dataset_info, encode_labels=True, max_memory_in_mb=max_mem)
    return D


# metalearning helpers
def _calculate_metafeatures(data_feat_type, data_info_task, basename,
                            x_train, y_train, watcher, logger):
    # == Calculate metafeatures
    task_name = 'CalculateMetafeatures'
    watcher.start_task(task_name)
    categorical = [True if feat_type.lower() in ['categorical'] else False
                   for feat_type in data_feat_type]

    EXCLUDE_META_FEATURES = EXCLUDE_META_FEATURES_CLASSIFICATION \
        if data_info_task in CLASSIFICATION_TASKS else EXCLUDE_META_FEATURES_REGRESSION

    if data_info_task in [MULTICLASS_CLASSIFICATION, BINARY_CLASSIFICATION,
                          MULTILABEL_CLASSIFICATION, REGRESSION]:
        logger.info('Start calculating metafeatures for %s', basename)
        result = calculate_all_metafeatures_with_labels(
            x_train, y_train, categorical=categorical,
            dataset_name=basename+SENTINEL,
            dont_calculate=EXCLUDE_META_FEATURES, )
        for key in list(result.metafeature_values.keys()):
            if result.metafeature_values[key].type_ != 'METAFEATURE':
                del result.metafeature_values[key]

    else:
        result = None
        logger.info('Metafeatures not calculated')
    watcher.stop_task(task_name)
    logger.info(
        'Calculating Metafeatures (categorical attributes) took %5.2f',
        watcher.wall_elapsed(task_name))
    return result

def _calculate_metafeatures_encoded(basename, x_train, y_train, watcher,
                                    task, logger):
    EXCLUDE_META_FEATURES = EXCLUDE_META_FEATURES_CLASSIFICATION \
        if task in CLASSIFICATION_TASKS else EXCLUDE_META_FEATURES_REGRESSION

    task_name = 'CalculateMetafeaturesEncoded'
    watcher.start_task(task_name)
    result = calculate_all_metafeatures_encoded_labels(
        x_train, y_train, categorical=[False] * x_train.shape[1],
        dataset_name=basename+SENTINEL, dont_calculate=EXCLUDE_META_FEATURES)
    for key in list(result.metafeature_values.keys()):
        if result.metafeature_values[key].type_ != 'METAFEATURE':
            del result.metafeature_values[key]
    watcher.stop_task(task_name)
    logger.info(
        'Calculating Metafeatures (encoded attributes) took %5.2fsec',
        watcher.wall_elapsed(task_name))
    return result

def _get_metalearning_configurations(meta_base, basename, metric,
                                     configuration_space,
                                     task,
                                     initial_configurations_via_metalearning,
                                     is_sparse,
                                     watcher, logger):
    task_name = 'InitialConfigurations'
    watcher.start_task(task_name)
    try:
        metalearning_configurations = suggest_via_metalearning(
            meta_base, basename, metric,
            task,
            is_sparse == 1,
            initial_configurations_via_metalearning
        )
    except Exception as e:
        logger.error("Error getting metalearning configurations!")
        logger.error(str(e))
        logger.error(traceback.format_exc())
        metalearning_configurations = []
    watcher.stop_task(task_name)
    return metalearning_configurations

def _print_debug_info_of_init_configuration(initial_configurations, basename,
                                            time_for_task, logger, watcher):
    logger.debug('Initial Configurations: (%d)' % len(initial_configurations))
    for initial_configuration in initial_configurations:
        logger.debug(initial_configuration)
    logger.debug('Looking for initial configurations took %5.2fsec',
                 watcher.wall_elapsed('InitialConfigurations'))
    logger.info(
        'Time left for %s after finding initial configurations: %5.2fsec',
        basename, time_for_task - watcher.wall_elapsed(basename))


class AutoMLScenario(Scenario):
    """
    We specialize the smac3 scenario here as we would like
    to create it in code, without actually reading a smac scenario file
    """

    def __init__(self, config_space, limit, cutoff_time, metafeatures):
        self.logger = get_logger(self.__class__.__name__)

        # Give SMAC at least 5 seconds
        soft_limit = max(5, cutoff_time - 35)

        scenario_dict = {'cs': config_space,
                         'run_obj': 'quality',
                         'cutoff': soft_limit,
                         'algo_runs_timelimit': soft_limit,
                         'wallclock-limit': limit,
                         'features': metafeatures,
                         'instances': [[name] for name in metafeatures]}

        super(AutoMLScenario, self).__init__(scenario_dict)

class AutoMLSMBO(multiprocessing.Process):

    def __init__(self, config_space, dataset_name,
                 output_dir, tmp_dir,
                 total_walltime_limit,
                 func_eval_time_limit,
                 memory_limit,
                 watcher, start_num_run=1,
                 data_memory_limit=None,
                 num_metalearning_cfgs=25,
                 config_file=None,
                 smac_iters=1000,
                 seed=1,
                 metadata_directory=None,
                 resampling_strategy='holdout',
                 resampling_strategy_args=None,
                 acquisition_function='EI'):
        super(AutoMLSMBO, self).__init__()
        # data related
        self.dataset_name = dataset_name
        self.output_dir = output_dir
        self.tmp_dir = tmp_dir
        self.datamanager = None
        self.metric = None
        self.task = None

        # the configuration space
        self.config_space = config_space

        # Evaluation
        self.resampling_strategy = resampling_strategy
        if resampling_strategy_args is None:
            resampling_strategy_args = {}
        self.resampling_strategy_args = resampling_strategy_args

        # and a bunch of useful limits
        self.total_walltime_limit = int(total_walltime_limit)
        self.func_eval_time_limit = int(func_eval_time_limit)
        self.memory_limit = memory_limit
        self.data_memory_limit = data_memory_limit
        self.watcher = watcher
        self.num_metalearning_cfgs = num_metalearning_cfgs
        self.config_file = config_file
        self.seed = seed
        self.metadata_directory = metadata_directory
        self.smac_iters = smac_iters
        self.start_num_run = start_num_run
        self.acquisition_function = acquisition_function

        self.config_space.seed(self.seed)
        logger_name = self.__class__.__name__ + \
                      (":" + dataset_name if dataset_name is not None else "")
        self.logger = get_logger(logger_name)

    def reset_data_manager(self, max_mem=None):
        if max_mem is None:
            max_mem = self.data_memory_limit
        if self.datamanager is not None:
            del self.datamanager
        if isinstance(self.dataset_name, AbstractDataManager):
            self.datamanager = self.dataset_name
        else:
            self.datamanager = load_data(self.dataset_name,
                                         self.output_dir,
                                         self.tmp_dir,
                                         max_mem = max_mem)
        self.metric = self.datamanager.info['metric']
        self.task = self.datamanager.info['task']
        
    def collect_additional_subset_defaults(self):
        default_configs = []
        # == set default configurations
        # first enqueue the default configuration from our config space
        if self.datamanager.info["task"] in CLASSIFICATION_TASKS:
            config_dict = {'balancing:strategy': 'weighting',
                           'classifier:__choice__': 'sgd',
                           'classifier:sgd:loss': 'hinge',
                           'classifier:sgd:penalty': 'l2',
                           'classifier:sgd:alpha': 0.0001,
                           'classifier:sgd:fit_intercept': 'True',
                           'classifier:sgd:n_iter': 5,
                           'classifier:sgd:learning_rate': 'optimal',
                           'classifier:sgd:eta0': 0.01,
                           'classifier:sgd:average': 'True',
                           'imputation:strategy': 'mean',
                           'one_hot_encoding:use_minimum_fraction': 'True',
                           'one_hot_encoding:minimum_fraction': 0.1,
                           'preprocessor:__choice__': 'no_preprocessing',
                           'rescaling:__choice__': 'min/max'}
            try:
                config = Configuration(self.config_space, config_dict)
                default_configs.append(config)
            except ValueError as e:
                self.logger.warning("Second default configurations %s cannot"
                                    " be evaluated because of %s" %
                                    (config_dict, e))

            if self.datamanager.info["is_sparse"]:
                config_dict = {'classifier:__choice__': 'extra_trees',
                               'classifier:extra_trees:bootstrap': 'False',
                               'classifier:extra_trees:criterion': 'gini',
                               'classifier:extra_trees:max_depth': 'None',
                               'classifier:extra_trees:max_features': 1.0,
                               'classifier:extra_trees:min_samples_leaf': 5,
                               'classifier:extra_trees:min_samples_split': 5,
                               'classifier:extra_trees:min_weight_fraction_leaf': 0.0,
                               'classifier:extra_trees:n_estimators': 100,
                               'balancing:strategy': 'weighting',
                               'imputation:strategy': 'mean',
                               'one_hot_encoding:use_minimum_fraction': 'True',
                               'one_hot_encoding:minimum_fraction': 0.1,
                               'preprocessor:__choice__': 'truncatedSVD',
                               'preprocessor:truncatedSVD:target_dim': 20,
                               'rescaling:__choice__': 'min/max'}
            else:
                n_data_points = self.datamanager.data['X_train'].shape[0]
                percentile = 20. / n_data_points
                percentile = max(percentile, 2.)

                config_dict = {'classifier:__choice__': 'extra_trees',
                               'classifier:extra_trees:bootstrap': 'False',
                               'classifier:extra_trees:criterion': 'gini',
                               'classifier:extra_trees:max_depth': 'None',
                               'classifier:extra_trees:max_features': 1.0,
                               'classifier:extra_trees:min_samples_leaf': 5,
                               'classifier:extra_trees:min_samples_split': 5,
                               'classifier:extra_trees:min_weight_fraction_leaf': 0.0,
                               'classifier:extra_trees:n_estimators': 100,
                               'balancing:strategy': 'weighting',
                               'imputation:strategy': 'mean',
                               'one_hot_encoding:use_minimum_fraction': 'True',
                               'one_hot_encoding:minimum_fraction': 0.1,
                               'preprocessor:__choice__': 'select_percentile_classification',
                               'preprocessor:select_percentile_classification:percentile': percentile,
                               'preprocessor:select_percentile_classification:score_func': 'chi2',
                               'rescaling:__choice__': 'min/max'}

            try:
                config = Configuration(self.config_space, config_dict)
                default_configs.append(config)
            except ValueError as e:
                self.logger.warning("Third default configurations %s cannot"
                                    " be evaluated because of %s" %
                                    (config_dict, e))

            if self.datamanager.info["is_sparse"]:
                config_dict = {'balancing:strategy': 'weighting',
                               'classifier:__choice__': 'multinomial_nb',
                               'classifier:multinomial_nb:alpha': 1.0,
                               'classifier:multinomial_nb:fit_prior': 'True',
                               'imputation:strategy': 'mean',
                               'one_hot_encoding:use_minimum_fraction': 'True',
                               'one_hot_encoding:minimum_fraction': 0.1,
                               'preprocessor:__choice__': 'no_preprocessing',
                               'rescaling:__choice__': 'none'}
            else:
                config_dict = {'balancing:strategy': 'weighting',
                               'classifier:__choice__': 'gaussian_nb',
                               'imputation:strategy': 'mean',
                               'one_hot_encoding:use_minimum_fraction': 'True',
                               'one_hot_encoding:minimum_fraction': 0.1,
                               'preprocessor:__choice__': 'no_preprocessing',
                               'rescaling:__choice__': 'standardize'}
            try:
                config = Configuration(self.config_space, config_dict)
                default_configs.append(config)
            except ValueError as e:
                self.logger.warning("Forth default configurations %s cannot"
                                    " be evaluated because of %s" %
                                    (config_dict, e))

        elif self.datamanager.info["task"] in REGRESSION_TASKS:
            config_dict = {'regressor:__choice__': 'sgd',
                           'regressor:sgd:loss': 'squared_loss',
                           'regressor:sgd:penalty': 'l2',
                           'regressor:sgd:alpha': 0.0001,
                           'regressor:sgd:fit_intercept': 'True',
                           'regressor:sgd:n_iter': 5,
                           'regressor:sgd:learning_rate': 'optimal',
                           'regressor:sgd:eta0': 0.01,
                           'regressor:sgd:average': 'True',
                           'imputation:strategy': 'mean',
                           'one_hot_encoding:use_minimum_fraction': 'True',
                           'one_hot_encoding:minimum_fraction': 0.1,
                           'preprocessor:__choice__': 'no_preprocessing',
                           'rescaling:__choice__': 'min/max'}
            try:
                config = Configuration(self.config_space, config_dict)
                default_configs.append(config)
            except ValueError as e:
                self.logger.warning("Second default configurations %s cannot"
                                    " be evaluated because of %s" %
                                    (config_dict, e))

            if self.datamanager.info["is_sparse"]:
                config_dict = {'regressor:__choice__': 'extra_trees',
                               'regressor:extra_trees:bootstrap': 'False',
                               'regressor:extra_trees:criterion': 'mse',
                               'regressor:extra_trees:max_depth': 'None',
                               'regressor:extra_trees:max_features': 1.0,
                               'regressor:extra_trees:min_samples_leaf': 5,
                               'regressor:extra_trees:min_samples_split': 5,
                               'regressor:extra_trees:n_estimators': 100,
                               'imputation:strategy': 'mean',
                               'one_hot_encoding:use_minimum_fraction': 'True',
                               'one_hot_encoding:minimum_fraction': 0.1,
                               'preprocessor:__choice__': 'truncatedSVD',
                               'preprocessor:truncatedSVD:target_dim': 10,
                               'rescaling:__choice__': 'min/max'}
            else:
                config_dict = {'regressor:__choice__': 'extra_trees',
                               'regressor:extra_trees:bootstrap': 'False',
                               'regressor:extra_trees:criterion': 'mse',
                               'regressor:extra_trees:max_depth': 'None',
                               'regressor:extra_trees:max_features': 1.0,
                               'regressor:extra_trees:min_samples_leaf': 5,
                               'regressor:extra_trees:min_samples_split': 5,
                               'regressor:extra_trees:n_estimators': 100,
                               'imputation:strategy': 'mean',
                               'one_hot_encoding:use_minimum_fraction': 'True',
                               'one_hot_encoding:minimum_fraction': 0.1,
                               'preprocessor:__choice__': 'pca',
                               'preprocessor:pca:keep_variance': 0.9,
                               'preprocessor:pca:whiten': 'False',
                               'rescaling:__choice__': 'min/max'}

            try:
                config = Configuration(self.config_space, config_dict)
                default_configs.append(config)
            except ValueError as e:
                self.logger.warning("Third default configurations %s cannot"
                                    " be evaluated because of %s" %
                                    (config_dict, e))

        else:
            self.logger.info("Tasktype unknown: %s" %
                             TASK_TYPES_TO_STRING[self.datamanager.info[
                                 "task"]])

        return default_configs

    def collect_metalearning_suggestions(self, meta_base):
        metalearning_configurations = _get_metalearning_configurations(
            meta_base=meta_base,
            basename=self.dataset_name+SENTINEL,
            metric=self.metric,
            configuration_space=self.config_space,
            task=self.task,
            is_sparse=self.datamanager.info['is_sparse'],
            initial_configurations_via_metalearning=self.num_metalearning_cfgs,
            watcher=self.watcher,
            logger=self.logger)
        _print_debug_info_of_init_configuration(
            metalearning_configurations,
            self.dataset_name,
            self.total_walltime_limit,
            self.logger,
            self.watcher)

        return metalearning_configurations

    def _calculate_metafeatures(self):
        meta_features = _calculate_metafeatures(
            data_feat_type=self.datamanager.feat_type,
            data_info_task=self.datamanager.info['task'],
            x_train=self.datamanager.data['X_train'],
            y_train=self.datamanager.data['Y_train'],
            basename=self.dataset_name,
            watcher=self.watcher,
            logger=self.logger)
        return meta_features

    def _calculate_metafeatures_with_limits(self, time_limit):
        res = None
        time_limit = max(time_limit, 1)
        try:
            safe_mf = pynisher.enforce_limits(mem_in_mb=self.memory_limit,
                                              wall_time_in_s=int(time_limit),
                                              grace_period_in_s=30)(
                self._calculate_metafeatures)
            res = safe_mf()
        except Exception as e:
            self.logger.error('Error getting metafeatures: %s', str(e))

        return res

    def _calculate_metafeatures_encoded(self):
        meta_features_encoded = _calculate_metafeatures_encoded(
            self.dataset_name,
            self.datamanager.data['X_train'],
            self.datamanager.data['Y_train'],
            self.watcher,
            self.datamanager.info['task'],
            self.logger)
        return meta_features_encoded

    def _calculate_metafeatures_encoded_with_limits(self, time_limit):
        res = None
        time_limit = max(time_limit, 1)
        try:
            safe_mf = pynisher.enforce_limits(mem_in_mb=self.memory_limit,
                                              wall_time_in_s=int(time_limit),
                                              grace_period_in_s=30)(
                self._calculate_metafeatures_encoded)
            res = safe_mf()
        except Exception as e:
            self.logger.error('Error getting metafeatures (encoded) : %s',
                              str(e))

        return res
    
    def run(self):
        # we use pynisher here to enforce limits
        safe_smbo = pynisher.enforce_limits(mem_in_mb=self.memory_limit,
                                            wall_time_in_s=int(self.total_walltime_limit),
                                            grace_period_in_s=5)(self.run_smbo)
        safe_smbo(max_iters = self.smac_iters)
        

    def run_smbo(self, max_iters=1000):
        global evaluator

        # == first things first: load the datamanager
        self.reset_data_manager()
        
        # == Initialize SMBO stuff
        # first create a scenario
        seed = self.seed # TODO
        num_params = len(self.config_space.get_hyperparameters())
        # allocate a run history
        run_history = RunHistory()
        meta_runhistory = RunHistory()
        meta_runs_dataset_indices = {}
        num_run = self.start_num_run
        instance_id = self.dataset_name + SENTINEL

        # == Train on subset
        #    before doing anything, let us run the default_cfg
        #    on a subset of the available data to ensure that
        #    we at least have some models
        #    we will try three different ratios of decreasing magnitude
        #    in the hope that at least on the last one we will be able
        #    to get a model
        n_data = self.datamanager.data['X_train'].shape[0]
        subset_ratio = 10000. / n_data
        if subset_ratio >= 0.5:
            subset_ratio = 0.33
            subset_ratios = [subset_ratio, subset_ratio * 0.10]
        else:
            subset_ratios = [subset_ratio, 500. / n_data]
        self.logger.info("Training default configurations on a subset of "
                         "%d/%d data points." %
                         (int(n_data * subset_ratio), n_data))

        # the time limit for these function evaluations is rigorously
        # set to only 1/2 of a full function evaluation
        subset_time_limit = max(5, int(self.func_eval_time_limit / 2))
        # the configs we want to run on the data subset are:
        # 1) the default configs
        # 2) a set of configs we selected for training on a subset
        subset_configs = [self.config_space.get_default_configuration()] \
                          + self.collect_additional_subset_defaults()
        subset_config_succesful = [False] * len(subset_configs)
        for subset_config_id, next_config in enumerate(subset_configs):
            for i, ratio in enumerate(subset_ratios):
                self.reset_data_manager()
                n_data_subsample = int(n_data * ratio)

                # run the config, but throw away the result afterwards
                # since this cfg was evaluated only on a subset
                # and we don't want  to confuse SMAC
                self.logger.info("Starting to evaluate %d on SUBSET "
                                 "with size %d and time limit %ds.",
                                 num_run, n_data_subsample,
                                 subset_time_limit)
                self.logger.info(next_config)
                _info = eval_with_limits(
                    self.datamanager, self.tmp_dir, next_config,
                    seed, num_run,
                    self.resampling_strategy,
                    self.resampling_strategy_args,
                    self.memory_limit,
                    subset_time_limit, n_data_subsample)
                (duration, result, _, additional_run_info, status) = _info
                self.logger.info("Finished evaluating %d. configuration on SUBSET. "
                                 "Duration %f; loss %f; status %s; additional run "
                                 "info: %s ", num_run, duration, result,
                                 str(status), additional_run_info)

                num_run += 1
                if i < len(subset_ratios) - 1:
                    if status != StatusType.SUCCESS:
                        # Do not increase num_run here, because we will try
                        # the same configuration with less data
                        self.logger.info("A CONFIG did not finish "
                                         " for subset ratio %f -> going smaller",
                                         ratio)
                        continue
                    else:
                        self.logger.info("Finished SUBSET training sucessfully "
                                         "with ratio %f", ratio)
                        subset_config_succesful[subset_config_id] = True
                        break
                else:
                    if status != StatusType.SUCCESS:
                        self.logger.info("A CONFIG did not finish "
                                         " for subset ratio %f.",
                                         ratio)
                        continue
                    else:
                        self.logger.info("Finished SUBSET training sucessfully "
                                         "with ratio %f", ratio)
                        subset_config_succesful[subset_config_id] = True
                        break

        # Use the first non-failing configuration from the subsets as the new
        #  default configuration -> this guards us against the random forest
        # failing on large, sparse datasets
        default_cfg = None
        for subset_config_id, next_config in enumerate(subset_configs):
            if subset_config_succesful[subset_config_id]:
                default_cfg = next_config
                break
        if default_cfg is None:
            default_cfg = self.config_space.get_default_configuration()

        # == METALEARNING suggestions
        # we start by evaluating the defaults on the full dataset again
        # and add the suggestions from metalearning behind it

        if self.metadata_directory is None:
            metalearning_directory = os.path.dirname(
                autosklearn.metalearning.__file__)
            # There is no multilabel data in OpenML
            if self.task == MULTILABEL_CLASSIFICATION:
                meta_task = BINARY_CLASSIFICATION
            else:
                meta_task = self.task
            metadata_directory = os.path.join(
                metalearning_directory, 'files',
                '%s_%s_%s' % (METRIC_TO_STRING[self.metric],
                              TASK_TYPES_TO_STRING[meta_task],
                              'sparse' if self.datamanager.info['is_sparse']
                              else 'dense'))
            self.metadata_directory = metadata_directory

        self.logger.info('Metadata directory: %s', self.metadata_directory)
        meta_base = MetaBase(self.config_space, self.metadata_directory)

        metafeature_calculation_time_limit = int(
            self.total_walltime_limit / 4)
        metafeature_calculation_start_time = time.time()
        meta_features = self._calculate_metafeatures_with_limits(
            metafeature_calculation_time_limit)
        metafeature_calculation_end_time = time.time()
        metafeature_calculation_time_limit = \
            metafeature_calculation_time_limit - (
            metafeature_calculation_end_time -
            metafeature_calculation_start_time)

        if metafeature_calculation_time_limit < 1:
            self.logger.warning('Time limit for metafeature calculation less '
                                'than 1 seconds (%f). Skipping calculation '
                                'of metafeatures for encoded dataset.',
                                metafeature_calculation_time_limit)
            meta_features_encoded = None
        else:
            self.datamanager.perform1HotEncoding()
            meta_features_encoded = \
                self._calculate_metafeatures_encoded_with_limits(
                    metafeature_calculation_time_limit)

        # In case there is a problem calculating the encoded meta-features
        if meta_features is None:
            if meta_features_encoded is not None:
                meta_features = meta_features_encoded
{'fraction_non_alphanumeric': 0.0745399115736381, 'fraction_numerical': 0.005011891270690013, 'mean_word_length': 1.7382867132867132, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 122, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '613975', 'n_tokens_mistral': 17056, 'n_tokens_neox': 16182, 'n_words': 3170}