<commit_before>import logging
import os
import jsonobject
import yaml


class _PillowAction(jsonobject.JsonObject):
    _allow_dynamic_properties = False
    include_groups = jsonobject.ListProperty(unicode, exclude_if_none=True)
    exclude_groups = jsonobject.ListProperty(unicode, exclude_if_none=True)
    include_pillows = jsonobject.ListProperty(unicode, exclude_if_none=True)
    exclude_pillows = jsonobject.ListProperty(unicode, exclude_if_none=True)


def apply_pillow_actions_to_pillows(pillow_actions, pillows_by_group):
    def coerce_to_pillow_action(obj):
        if isinstance(obj, _PillowAction):
            return obj
        return _PillowAction(obj)

    return _apply_pillow_actions_to_pillows(map(coerce_to_pillow_action, pillow_actions), pillows_by_group)


def _get_pillow_configs_from_settings_dict(pillows_by_group):
    # this sucks, but not sure there's a better way to make it available to fabric
    from manage import init_hq_python_path
    init_hq_python_path()
    from pillowtop.utils import get_pillow_configs_from_settings_dict
    return get_pillow_configs_from_settings_dict(pillows_by_group)


def _apply_pillow_actions_to_pillows(pillow_actions, pillows_by_group):
    def is_relevant(pillow_actions, pillow_config):
        # the default is to include if nothing specified
        relevant = True
        # the order of these checks is important since the actions are resolved in the order they are passed in
        for action in pillow_actions:
            if pillow_config.section in action.include_groups:
                assert pillow_config.section not in action.exclude_groups
                relevant = True
            if pillow_config.section in action.exclude_groups:
                relevant = False
            if pillow_config.class_name in action.include_pillows:
                assert pillow_config.class_name not in action.exclude_pillows
                relevant = True
            if pillow_config.class_name in action.exclude_pillows:<commit_msg>support including/excluding pillows by name instead of class<commit_after>import logging
import os
import jsonobject
import yaml


class _PillowAction(jsonobject.JsonObject):
    _allow_dynamic_properties = False
    include_groups = jsonobject.ListProperty(unicode, exclude_if_none=True)
    exclude_groups = jsonobject.ListProperty(unicode, exclude_if_none=True)
    include_pillows = jsonobject.ListProperty(unicode, exclude_if_none=True)
    exclude_pillows = jsonobject.ListProperty(unicode, exclude_if_none=True)


def apply_pillow_actions_to_pillows(pillow_actions, pillows_by_group):
    def coerce_to_pillow_action(obj):
        if isinstance(obj, _PillowAction):
            return obj
        return _PillowAction(obj)

    return _apply_pillow_actions_to_pillows(map(coerce_to_pillow_action, pillow_actions), pillows_by_group)


def _get_pillow_configs_from_settings_dict(pillows_by_group):
    # this sucks, but not sure there's a better way to make it available to fabric
    from manage import init_hq_python_path
    init_hq_python_path()
    from pillowtop.utils import get_pillow_configs_from_settings_dict
    return get_pillow_configs_from_settings_dict(pillows_by_group)


def _apply_pillow_actions_to_pillows(pillow_actions, pillows_by_group):
    def is_relevant(pillow_actions, pillow_config):
        # the default is to include if nothing specified
        relevant = True
        # the order of these checks is important since the actions are resolved in the order they are passed in
        def _is_a_match(p_config, list_of_pillows):
            return (
                p_config.class_name in list_of_pillows or
                p_config.name in list_of_pillows
            )
        for action in pillow_actions:
            if pillow_config.section in action.include_groups:
                assert pillow_config.section not in action.exclude_groups
                relevant = True
            if pillow_config.section in action.exclude_groups:
                relevant = False
            if _is_a_match(pillow_config, action.include_pillows):
                assert pillow_config.class_name not in action.exclude_pillows
                relevant = True
            if _is_a_match(pillow_config, action.exclude_pillows):
{'fraction_non_alphanumeric': 0.04670265195963389, 'fraction_numerical': 0.0, 'mean_word_length': 3.6427015250544663, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6598149', 'n_tokens_mistral': 1301, 'n_tokens_neox': 1251, 'n_words': 293}