<commit_before>from __future__ import unicode_literals, division, absolute_import
import copy
import os
import logging

from flexget.config_schema import one_or_more
from flexget.plugin import register_plugin, priority, PluginWarning
from flexget.utils.log import log_once
from flexget.utils.titles import ParseWarning

log = logging.getLogger('exists_series')


class FilterExistsSeries(object):
    """
    Intelligent series aware exists rejecting.

    Example::

      exists_series: /storage/series/
    """

    schema = {
        'anyOf': [
            one_or_more({'type': 'string', 'format': 'path'}),
            {
                'type': 'object',
                'properties': {
                    'path': one_or_more({'type': 'string', 'format': 'path'}),
                    'allow_different_qualities': {'enum': ['better', True, False], 'default': False}
                },
                'required': ['path'],
                'additionalProperties': False
            }
        ]
    }

    def get_config(self, task):
        config = task.config.get('exists_series', [])
        # if config is not a dict, assign value to 'path' key
        if not isinstance(config, dict):
            config = {'path': config}
        # if only a single path is passed turn it into a 1 element list
        if isinstance(config['path'], basestring):
            config['path'] = [config['path']]
        return config

    @priority(-1)
    def on_task_filter(self, task):
        if not task.accepted:
            log.debug('Scanning not needed')
            return
        config = self.get_config(task)
        accepted_series = {}
        paths = set()
        for entry in task.accepted:
            if 'series_parser' in entry:
                if entry['series_parser'].valid:
                    accepted_series.setdefault(entry['series_parser'].name, []).append(entry)
                    for path in config['path']:
                        paths.add(entry.render(path))
                else:
                    log.debug('entry %s series_parser invalid', entry['title'])
        if not accepted_series:
            log.warning('No accepted entries have series information. exists_series cannot filter them')
            return

        for path in paths:
            log.verbose('Scanning %s', path)
            # crashes on some paths with unicode
            path = str(os.path.expanduser(path))
            if not os.path.exists(path):
                raise PluginWarning('Path %s does not exist' % path, log)
            # scan through
            for root, dirs, files in os.walk(path):
                # convert filelists into utf-8 to avoid unicode problems
                dirs = [x.decode('utf-8', 'ignore') for x in dirs]
                files = [x.decode('utf-8', 'ignore') for x in files]
                # For speed, only test accepted entries since our priority should be after everything is accepted.<commit_msg>Catch RenderErrors in exists_series<commit_after>from __future__ import unicode_literals, division, absolute_import
import copy
import os
import logging

from flexget.config_schema import one_or_more
from flexget.plugin import register_plugin, priority, PluginWarning
from flexget.utils.log import log_once
from flexget.utils.template import RenderError
from flexget.utils.titles import ParseWarning

log = logging.getLogger('exists_series')


class FilterExistsSeries(object):
    """
    Intelligent series aware exists rejecting.

    Example::

      exists_series: /storage/series/
    """

    schema = {
        'anyOf': [
            one_or_more({'type': 'string', 'format': 'path'}),
            {
                'type': 'object',
                'properties': {
                    'path': one_or_more({'type': 'string', 'format': 'path'}),
                    'allow_different_qualities': {'enum': ['better', True, False], 'default': False}
                },
                'required': ['path'],
                'additionalProperties': False
            }
        ]
    }

    def get_config(self, task):
        config = task.config.get('exists_series', [])
        # if config is not a dict, assign value to 'path' key
        if not isinstance(config, dict):
            config = {'path': config}
        # if only a single path is passed turn it into a 1 element list
        if isinstance(config['path'], basestring):
            config['path'] = [config['path']]
        return config

    @priority(-1)
    def on_task_filter(self, task):
        if not task.accepted:
            log.debug('Scanning not needed')
            return
        config = self.get_config(task)
        accepted_series = {}
        paths = set()
        for entry in task.accepted:
            if 'series_parser' in entry:
                if entry['series_parser'].valid:
                    accepted_series.setdefault(entry['series_parser'].name, []).append(entry)
                    for path in config['path']:
                        try:
                            paths.add(entry.render(path))
                        except RenderError as e:
                            log.error('Error rendering path `%s`: %s', path, e)
                else:
                    log.debug('entry %s series_parser invalid', entry['title'])
        if not accepted_series:
            log.warning('No accepted entries have series information. exists_series cannot filter them')
            return

        for path in paths:
            log.verbose('Scanning %s', path)
            # crashes on some paths with unicode
            path = str(os.path.expanduser(path))
            if not os.path.exists(path):
                raise PluginWarning('Path %s does not exist' % path, log)
            # scan through
            for root, dirs, files in os.walk(path):
                # convert filelists into utf-8 to avoid unicode problems
                dirs = [x.decode('utf-8', 'ignore') for x in dirs]
                files = [x.decode('utf-8', 'ignore') for x in files]
                # For speed, only test accepted entries since our priority should be after everything is accepted.
{'fraction_non_alphanumeric': 0.1049534085335949, 'fraction_numerical': 0.0016347882949158084, 'mean_word_length': 1.956984050265829, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7543793', 'n_tokens_mistral': 1618, 'n_tokens_neox': 1548, 'n_words': 490}