<commit_before>from __future__ import unicode_literals, division, absolute_import
import logging

from flexget import plugin
from flexget.event import event

log = logging.getLogger('urlrewrite_redirect')


class UrlRewriteRedirect(object):
    """Rewrites urls which actually redirect somewhere else."""
    def __init__(self):
        self.processed = set()

    def on_task_start(self):
        self.processed = set()

    def url_rewritable(self, task, entry):
        if not any(entry['url'].startswith(adapter) for adapter in task.requests.adapters):
            return False
        return entry['url'] not in self.processed

    def url_rewrite(self, task, entry):
        # Don't accidentally go online in unit tests
        if task.manager.unit_test:
            return
        auth = None
        if 'download_auth' in entry:
            auth = entry['download_auth']
            log.debug('Custom auth enabled for %s url_redirect: %s' % (entry['title'], entry['download_auth']))
        try:
            r = task.requests.head(entry['url'], auth=auth, allow_redirects=True)
        except Exception:
            pass
        else:
            if r.status_code < 400 and r.url != entry['url']:
                entry['url'] = r.url
        finally:
            # Make sure we don't try to rewrite this url again
            self.processed.add(entry['url'])


@event('plugin.register')
def register_plugin():
    plugin.register(UrlRewriteRedirect, 'urlrewrite_redirect', groups=['urlrewriter'], api_ver=2)<commit_msg>Fix issue with urlrewrite_redirect in unit tests from last commit<commit_after>from __future__ import unicode_literals, division, absolute_import
import logging

from flexget import plugin
from flexget.event import event

log = logging.getLogger('urlrewrite_redirect')


class UrlRewriteRedirect(object):
    """Rewrites urls which actually redirect somewhere else."""
    def __init__(self):
        self.processed = set()

    def on_task_start(self):
        self.processed = set()

    def url_rewritable(self, task, entry):
        if not any(entry['url'].startswith(adapter) for adapter in task.requests.adapters):
            return False
        return entry['url'] not in self.processed

    def url_rewrite(self, task, entry):
        # Don't accidentally go online in unit tests
        if not task.manager.unit_test:
            auth = None
            if 'download_auth' in entry:
                auth = entry['download_auth']
                log.debug('Custom auth enabled for %s url_redirect: %s' % (entry['title'], entry['download_auth']))
            try:
                r = task.requests.head(entry['url'], auth=auth, allow_redirects=True)
            except Exception:
                pass
            else:
                if r.status_code < 400 and r.url != entry['url']:
                    entry['url'] = r.url
        # Make sure we don't try to rewrite this url again
        self.processed.add(entry['url'])


@event('plugin.register')
def register_plugin():
    plugin.register(UrlRewriteRedirect, 'urlrewrite_redirect', groups=['urlrewriter'], api_ver=2)
{'fraction_non_alphanumeric': 0.1045016077170418, 'fraction_numerical': 0.002572347266881029, 'mean_word_length': 2.725748502994012, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4877024', 'n_tokens_mistral': 887, 'n_tokens_neox': 833, 'n_words': 247}