<commit_before>from __future__ import unicode_literals, division, absolute_import
import urllib
import logging
import re

from flexget.entry import Entry
from flexget.plugin import register_plugin, priority, get_plugin_by_name

log = logging.getLogger('regexp')


class FilterRegexp(object):

    """
        All possible forms.

        regexp:
          [operation]:           # operation to perform on matches
            - [regexp]           # simple regexp
            - [regexp]: <path>   # override path
            - [regexp]:
                [path]: <path>   # override path
                [not]: <regexp>  # not match
                [from]: <field>  # search from given entry field
            - [regexp]:
                [path]: <path>   # override path
                [not]:           # list of not match regexps
                  - <regexp>
                [from]:          # search only from these fields
                  - <field>
          [operation]:
            - <regexp>
          [rest]: <operation>    # non matching entries are
          [from]:                # search only from these fields for all regexps
            - <field>

        Possible operations: accept, reject, accept_excluding, reject_excluding
    """

    def validator(self):
        from flexget import validator

        def build_list(regexps):
            regexps.accept('regexp')

            # bundle is a dictionary form
            bundle = regexps.accept('dict')
            value_validator = bundle.accept_valid_keys('root', key_type='regexp')
            # path as a single parameter
            value_validator.accept('path', allow_replacement=True)

            # advanced configuration as a parameter
            advanced = value_validator.accept('dict')
            advanced.accept('path', key='path', allow_replacement=True)
            # accept set parameters
            set = advanced.accept('dict', key='set')
            set.accept_any_key('any')
            # not as a single parameter
            advanced.accept('regexp', key='not')
            # not in a list form
            advanced.accept('list', key='not').accept('regexp')
            # from as a single parameter
            advanced.accept('text', key='from')
            # from in a list form
            advanced.accept('list', key='from').accept('text')

        conf = validator.factory('dict')
        for operation in ['accept', 'reject', 'accept_excluding', 'reject_excluding']:
            regexps = conf.accept('list', key=operation)
            build_list(regexps)

        conf.accept('choice', key='rest').accept_choices(['accept', 'reject'])
        conf.accept('text', key='from')
        conf.accept('list', key='from').accept('text')
        return conf

    def prepare_config(self, config):
        """Returns the config in standard format.

        All regexps are turned into dictionaries in the form of {compiled regexp: options}

        :param config: Dict that can optionally contain the following keys
            path: will be attached to entries that match
            set: a dict of values to be attached to entries that match via set plugin
            from: a list of fields in entry for the regexps to match against
            not: a list of compiled regexps that if matching, will disqualify the main match
        :return: New config dictionary
        """
        out_config = {}
        if 'rest' in config:
            out_config['rest'] = config['rest']
        # Turn all our regexps into advanced form dicts and compile them
        for operation, regexps in config.iteritems():
            if operation in ['rest', 'from']:
                continue
            for regexp_item in regexps:
                if not isinstance(regexp_item, dict):
                    regexp = regexp_item<commit_msg>Convert regexp plugin to new schema<commit_after>from __future__ import unicode_literals, division, absolute_import
import urllib
import logging
import re

from flexget.config_schema import one_or_more
from flexget.entry import Entry
from flexget.plugin import register_plugin, priority, get_plugin_by_name

log = logging.getLogger('regexp')


class FilterRegexp(object):

    """
        All possible forms.

        regexp:
          [operation]:           # operation to perform on matches
            - [regexp]           # simple regexp
            - [regexp]: <path>   # override path
            - [regexp]:
                [path]: <path>   # override path
                [not]: <regexp>  # not match
                [from]: <field>  # search from given entry field
            - [regexp]:
                [path]: <path>   # override path
                [not]:           # list of not match regexps
                  - <regexp>
                [from]:          # search only from these fields
                  - <field>
          [operation]:
            - <regexp>
          [rest]: <operation>    # non matching entries are
          [from]:                # search only from these fields for all regexps
            - <field>

        Possible operations: accept, reject, accept_excluding, reject_excluding
    """

    schema = {
        'type': 'object',
        'properties': {
            'accept': {'$ref': '#/definitions/regex_list'},
            'reject': {'$ref': '#/definitions/regex_list'},
            'accept_excluding': {'$ref': '#/definitions/regex_list'},
            'reject_excluding': {'$ref': '#/definitions/regex_list'},
            'rest': {'type': 'string', 'enum': ['accept', 'reject']},
            'from': one_or_more({'type': 'string'})
        },
        'additionalProperties': False,
        'definitions': {
            # The validator for a list of regexps, each with or without settings
            'regex_list': {
                'type': 'array',
                'items': {
                    'oneOf': [
                        # Plain regex string
                        {'type': 'string', 'format': 'regex'},
                        # Regex with options (regex is key, options are value)
                        {
                            'type': 'object',
                            'additionalProperties': {
                                'oneOf': [
                                    # Simple options, just path
                                    {'type': 'string', 'format': 'path'},
                                    # Dict style options
                                    {
                                        'type': 'object',
                                        'properties': {
                                            'path': {'type': 'string', 'format': 'path'},
                                            'set': {'type': 'object'},
                                            'not': one_or_more({'type': 'string', 'format': 'regex'}),
                                            'from': one_or_more({'type': 'string'})
                                        },
                                        'additionalProperties': False
                                    }
                                ]
                            }
                        }
                    ]
                }
            }
        }
    }

    def prepare_config(self, config):
        """Returns the config in standard format.

        All regexps are turned into dictionaries in the form of {compiled regexp: options}

        :param config: Dict that can optionally contain the following keys
            path: will be attached to entries that match
            set: a dict of values to be attached to entries that match via set plugin
            from: a list of fields in entry for the regexps to match against
            not: a list of compiled regexps that if matching, will disqualify the main match
        :return: New config dictionary
        """
        out_config = {}
        if 'rest' in config:
            out_config['rest'] = config['rest']
        # Turn all our regexps into advanced form dicts and compile them
        for operation, regexps in config.iteritems():
            if operation in ['rest', 'from']:
                continue
            for regexp_item in regexps:
                if not isinstance(regexp_item, dict):
                    regexp = regexp_item
{'fraction_non_alphanumeric': 0.09657170449058426, 'fraction_numerical': 0.0, 'mean_word_length': 1.5037775763070413, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '499895', 'n_tokens_mistral': 1999, 'n_tokens_neox': 1917, 'n_words': 652}