<commit_before>"""
YAML parser for select config files
"""

# from standard library
import os
import copy

# from dependencies
import yaml
import numpy as np
from sklearn.externals import joblib

# from hvc
from ..features.extract import neural_net_features_switch_case_dict

path = os.path.abspath(__file__)  # get the path of this file
dir_path = os.path.dirname(path)  # but then just take the dir

with open(os.path.join(dir_path, 'validation.yml')) as val_yaml:
    validate_dict = yaml.load(val_yaml)

VALID_MODEL_KEYS = validate_dict['valid_model_keys']
VALID_HYPERPARAMS = validate_dict['hyperparameters']

# valid_models is a dict, iterate to concatenate all model names
# keys are model subtypes, i.e. sklearn and keras
# values are lists of valid model names
VALID_MODELS = []
for model_subtype in validate_dict['valid_models'].values():
    # iterate over list to append
    for model_name in model_subtype:
        VALID_MODELS.append(model_name)
VALID_MODELS = set(VALID_MODELS)

# also need valid_models as a dict, to determine whether
# a given model is in scikit-learn or in keras
MODEL_TYPES = validate_dict['valid_models']

with open(os.path.join(dir_path, 'feature_groups.yml')) as ftr_grp_yaml:
    valid_feature_groups_dict = yaml.load(ftr_grp_yaml)
VALID_FEATURE_GROUPS = set(valid_feature_groups_dict.keys())


def _validate_model_dict(model_dict,
                         index,
                         ftr_grp_ID_dict=None,
                         ftr_list_group_ID=None):
    """validates model dictionaries from 'models' list

    Parameters
    ----------
    model_dict : dict
    index : int
    ftr_grp_ID_dict : dict
        from feature file. validate_yaml checks whether it is
        defined in feature_file and if so passes as an argument.
        Default is None.
    ftr_list_group_ID : numpy 1-d vector
        from feature file. validate_yaml checks whether it is
        defined in feature_file and if so passes as an argument.
        Default is None.

    Returns
    -------
    validated_model_dict : dict
    """

    # no need to validate value of 'model_name' key
    # because that was done in _validate_models function
    # before calling _validate_model_dict.
    # so now check all the other keys + values
    model_dict_keys = set(model_dict.keys()) - {'model_name'}
    # get list of valid keys by using model name as key for VALID_MODEL_KEYS dict
    valid_keys_for_this_model = set(VALID_MODEL_KEYS[model_dict['model_name']])
    if not model_dict_keys.issubset(valid_keys_for_this_model):
        invalid_keys = model_dict_keys - valid_keys_for_this_model
        raise KeyError('Found invalid keys in item {0} from '
                       'list of model dictionaries: {1}'
                       .format(index, invalid_keys))

    validated_model_dict = copy.deepcopy(model_dict)

    if model_dict['model_name'] in MODEL_TYPES['sklearn']:
        # throw an error if both feature_list_indices and
        # feature_group are defined as keys for model dict
        if 'feature_list_indices' in model_dict \
                and 'feature_group' in model_dict:
            raise KeyError('both feature_list_indices and'
                           'feature_group defined in model_dict, not'
                           'clear which to use.')

        # throw an error if neither feature_list_indices nor
        # feature_group are defined as keys for model dict
        if 'feature_list_indices' not in model_dict \
                and 'feature_group' not in model_dict:
            raise KeyError('Neither feature_list_indices or '
                           'feature_group are defined in model_dict,'
                           'at least one of them must be defined.')

        if 'feature_list_indices' in model_dict:
            ftr_list_inds = model_dict['feature_list_indices']
            if type(ftr_list_inds) != list and type(ftr_list_inds) != str:
                raise ValueError('\'feature_list_indices\' should be a list or string '
                                 'but parsed as a {}'
                                 .format(type(ftr_list_inds)))
            if type(ftr_list_inds) == str:
                if ftr_list_inds == 'all':
                    pass  # just keep as 'all'
                else:
                    try:
                        ftr_list_inds = [int(num) for num in model_val.split(',')]
                    except ValueError:
                        raise ValueError('feature_list_indices parsed as a string '
                                         'but could not convert following to list of ints: {}'
                                         .format(ftr_list_inds))

            if ftr_list_inds != 'all':
                if not all([type(item_val) is int for item_val in ftr_list_inds]):
                    raise ValueError('all indices in \'feature_list_indices\''
                                     ' should be integers')
            validated_model_dict['feature_list_indices'] = ftr_list_inds

        if 'feature_group' in model_dict:
            ftr_grp = model_dict['feature_group']
            # feature group should work as a string or as a list of strings
            if type(ftr_grp) != str and type(ftr_grp) != list:
                raise ValueError('value for feature_group should'
                                 'be string but parsed as {}'
                                 .format(type(ftr_grp)))

            if type(ftr_grp) == str:
                if ftr_grp not in VALID_FEATURE_GROUPS:
                    raise ValueError('{} is not a valid feature group.'
                                     .format(ftr_grp))
                # get appropriate ID # out of ftr_grp_ID_dict for this model
                # (if called from todo_list so we hve ftr_list_group_ID/dict)
                if ftr_list_group_ID is not None and ftr_grp_ID_dict is not None:
                    ftr_grp_ID = ftr_grp_ID_dict[model_dict['model_name']]
                    # now find all the indices of features associated with the
                    # feature group for that model
                    ftr_inds = np.where(
                        np.in1d(ftr_list_group_ID, ftr_grp_ID))[0]  # returns tuple
                else:
                    ftr_inds = None

                if ftr_inds is not None:
                    validated_model_dict['feature_list_indices'] = ftr_inds

            elif type(ftr_grp) == list:
                if not all([type(item) == str for item in ftr_grp]):
                    raise ValueError('Not all items in feature_group list are strings.')
                ftr_inds = []
                if not all([model_name in VALID_FEATURE_GROUPS
                            for model_name in ftr_grp]):
                    raise ValueError('{} is not a valid feature group.'
                                     .format(ftr_grp))
                # if called from todo_list so we hve ftr_list_group_ID/dict
                if ftr_list_group_ID is not None and ftr_grp_ID_dict is not None:
                    for model_name in ftr_grp:
                        # get appropriate ID<commit_msg>clarify error message in hvc.parse.select<commit_after>"""
YAML parser for select config files
"""

# from standard library
import os
import copy

# from dependencies
import yaml
import numpy as np
from sklearn.externals import joblib

# from hvc
from ..features.extract import neural_net_features_switch_case_dict

path = os.path.abspath(__file__)  # get the path of this file
dir_path = os.path.dirname(path)  # but then just take the dir

with open(os.path.join(dir_path, 'validation.yml')) as val_yaml:
    validate_dict = yaml.load(val_yaml)

VALID_MODEL_KEYS = validate_dict['valid_model_keys']
VALID_HYPERPARAMS = validate_dict['hyperparameters']

# valid_models is a dict, iterate to concatenate all model names
# keys are model subtypes, i.e. sklearn and keras
# values are lists of valid model names
VALID_MODELS = []
for model_subtype in validate_dict['valid_models'].values():
    # iterate over list to append
    for model_name in model_subtype:
        VALID_MODELS.append(model_name)
VALID_MODELS = set(VALID_MODELS)

# also need valid_models as a dict, to determine whether
# a given model is in scikit-learn or in keras
MODEL_TYPES = validate_dict['valid_models']

with open(os.path.join(dir_path, 'feature_groups.yml')) as ftr_grp_yaml:
    valid_feature_groups_dict = yaml.load(ftr_grp_yaml)
VALID_FEATURE_GROUPS = set(valid_feature_groups_dict.keys())


def _validate_model_dict(model_dict,
                         index,
                         ftr_grp_ID_dict=None,
                         ftr_list_group_ID=None):
    """validates model dictionaries from 'models' list

    Parameters
    ----------
    model_dict : dict
    index : int
        index of this dict in list of dicts,
        used in error messages
    ftr_grp_ID_dict : dict
        from feature file. validate_yaml checks whether it is
        defined in feature_file and if so passes as an argument.
        Default is None.
    ftr_list_group_ID : numpy 1-d vector
        from feature file. validate_yaml checks whether it is
        defined in feature_file and if so passes as an argument.
        Default is None.

    Returns
    -------
    validated_model_dict : dict
    """

    # no need to validate value of 'model_name' key
    # because that was done in _validate_models function
    # before calling _validate_model_dict.
    # so now check all the other keys + values
    model_dict_keys = set(model_dict.keys()) - {'model_name'}
    # get list of valid keys by using model name as key for VALID_MODEL_KEYS dict
    valid_keys_for_this_model = set(VALID_MODEL_KEYS[model_dict['model_name']])
    if not model_dict_keys.issubset(valid_keys_for_this_model):
        invalid_keys = model_dict_keys - valid_keys_for_this_model
        raise KeyError('Found invalid keys in item {0} from '
                       'list of model dictionaries: {1}'
                       .format(index, invalid_keys))

    validated_model_dict = copy.deepcopy(model_dict)

    if model_dict['model_name'] in MODEL_TYPES['sklearn']:
        # throw an error if both feature_list_indices and
        # feature_group are defined as keys for model dict
        if 'feature_list_indices' in model_dict \
                and 'feature_group' in model_dict:
            raise KeyError('both feature_list_indices and'
                           'feature_group defined in model_dict, not'
                           'clear which to use.')

        # throw an error if neither feature_list_indices nor
        # feature_group are defined as keys for model dict
        if 'feature_list_indices' not in model_dict \
                and 'feature_group' not in model_dict:
            raise KeyError('Neither feature_list_indices or '
                           'feature_group are defined in model_dict,'
                           'at least one of them must be defined.')

        if 'feature_list_indices' in model_dict:
            ftr_list_inds = model_dict['feature_list_indices']
            if type(ftr_list_inds) != list and type(ftr_list_inds) != str:
                raise ValueError('\'feature_list_indices\' should be a list or string '
                                 'but parsed as a {}'
                                 .format(type(ftr_list_inds)))
            if type(ftr_list_inds) == str:
                if ftr_list_inds == 'all':
                    pass  # just keep as 'all'
                else:
                    try:
                        ftr_list_inds = [int(num) for num in model_val.split(',')]
                    except ValueError:
                        raise ValueError('feature_list_indices parsed as a string '
                                         'but could not convert following to list of ints: {}'
                                         .format(ftr_list_inds))

            if ftr_list_inds != 'all':
                if not all([type(item_val) is int for item_val in ftr_list_inds]):
                    raise ValueError('all indices in \'feature_list_indices\''
                                     ' should be integers')
            validated_model_dict['feature_list_indices'] = ftr_list_inds

        if 'feature_group' in model_dict:
            ftr_grp = model_dict['feature_group']
            # feature group should work as a string or as a list of strings
            if type(ftr_grp) != str and type(ftr_grp) != list:
                raise ValueError('value for feature_group should be '
                                 'string or list of strings but parsed as {}'
                                 .format(type(ftr_grp)))

            if type(ftr_grp) == str:
                if ftr_grp not in VALID_FEATURE_GROUPS:
                    raise ValueError('{} is not a valid feature group.'
                                     .format(ftr_grp))
                # get appropriate ID # out of ftr_grp_ID_dict for this model
                # (if called from todo_list so we have ftr_list_group_ID/dict)
                if ftr_list_group_ID is not None and ftr_grp_ID_dict is not None:
                    ftr_grp_ID = ftr_grp_ID_dict[model_dict['model_name']]
                    # now find all the indices of features associated with the
                    # feature group for that model
                    ftr_inds = np.where(
                        np.in1d(ftr_list_group_ID, ftr_grp_ID))[0]  # returns tuple
                else:
                    ftr_inds = None

                if ftr_inds is not None:
                    validated_model_dict['feature_list_indices'] = ftr_inds

            elif type(ftr_grp) == list:
                if not all([type(item) == str for item in ftr_grp]):
                    raise ValueError('Not all items in feature_group list are strings.')
                ftr_inds = []
                if not all([model_name in VALID_FEATURE_GROUPS
                            for model_name in ftr_grp]):
                    raise ValueError('{} is not a valid feature group.'
                                     .format(ftr_grp))
                # if called from todo_list so we hve ftr_list_group_ID/dict
                if ftr_list_group_ID is not None and ftr_grp_ID_dict is not None:
                    for model_name in ftr_grp:
                        # get appropriate ID
{'fraction_non_alphanumeric': 0.06660171201892964, 'fraction_numerical': 0.0006959426543252836, 'mean_word_length': 1.9434657927079066, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4469339', 'n_tokens_mistral': 4045, 'n_tokens_neox': 3963, 'n_words': 1235}