<commit_before>from __future__ import unicode_literals, division, absolute_import
import re
import logging
from urlparse import urlparse, urlunparse
from requests import RequestException

from flexget import plugin
from flexget.event import event
from flexget.plugins.plugin_urlrewriting import UrlRewritingError
from flexget.utils import requests
from flexget.utils.soup import get_soup

log = logging.getLogger('eztv')

EZTV_MIRRORS = [
    ('http', 'eztv.it'),
    ('https', 'eztv-proxy.net'),
    ('http', 'eztv.come.in')]


class UrlRewriteEztv(object):
    """Eztv url rewriter."""

    def url_rewritable(self, task, entry):
        return urlparse(entry['url']).netloc == 'eztv.it'

    def url_rewrite(self, task, entry):
        url = entry['url']
        for (scheme, netloc) in EZTV_MIRRORS:
            try:
                _, _, path, params, query, fragment = urlparse(url)
                url = urlunparse((scheme, netloc, path, params, query, fragment))
                page = requests.get(url).content
            except RequestException as e:
                log.debug('Eztv mirror `%s` seems to be down', url)
                continue
            break

        if not page:
            raise UrlRewritingError('No mirrors found for url %s' % entry['url'])

        log.debug('Eztv mirror `%s` chosen', url)
        try:
            soup = get_soup(page)
            mirrors = soup.find('a', attrs={'class': re.compile(r'download_\d')})
            if not mirrors:
                raise UrlRewritingError('Unable to locate download link from url %s'
                                        % url)
            entry['urls'] = [m.get('href') for m in mirrors]
            entry['url'] = mirrors[0].get('href')
        except Exception as e:
            raise UrlRewritingError(e)


@event('plugin.register')
def register_plugin():
    plugin.register(UrlRewriteEztv, 'eztv', groups=['urlrewriter'], api_ver=2)<commit_msg>Add url rewriter for eztv<commit_after>from __future__ import unicode_literals, division, absolute_import
import re
import logging
from urlparse import urlparse, urlunparse
from requests import RequestException

from flexget import plugin
from flexget.event import event
from flexget.plugins.plugin_urlrewriting import UrlRewritingError
from flexget.utils import requests
from flexget.utils.soup import get_soup

log = logging.getLogger('eztv')

EZTV_MIRRORS = [
    ('http', 'eztv.it'),
    ('https', 'eztv-proxy.net'),
    ('http', 'eztv.come.in')]


class UrlRewriteEztv(object):
    """Eztv url rewriter."""

    def url_rewritable(self, task, entry):
        return urlparse(entry['url']).netloc == 'eztv.it'

    def url_rewrite(self, task, entry):
        url = entry['url']
        page = None
        for (scheme, netloc) in EZTV_MIRRORS:
            try:
                _, _, path, params, query, fragment = urlparse(url)
                url = urlunparse((scheme, netloc, path, params, query, fragment))
                page = requests.get(url).content
            except RequestException as e:
                log.debug('Eztv mirror `%s` seems to be down', url)
                continue
            break

        if not page:
            raise UrlRewritingError('No mirrors found for url %s' % entry['url'])

        log.debug('Eztv mirror `%s` chosen', url)
        try:
            soup = get_soup(page)
            mirrors = soup.find_all('a', attrs={'class': re.compile(r'download_\d')})
        except Exception as e:
            raise UrlRewritingError(e)

        log.debug('%d torrent mirrors found', len(mirrors))

        if not mirrors:
            raise UrlRewritingError('Unable to locate download link from url %s' % url)

        entry['urls'] = [m.get('href') for m in mirrors]
        entry['url'] = mirrors[0].get('href')



@event('plugin.register')
def register_plugin():
    plugin.register(UrlRewriteEztv, 'eztv', groups=['urlrewriter'], api_ver=2)
{'fraction_non_alphanumeric': 0.11677715451300356, 'fraction_numerical': 0.0010198878123406426, 'mean_word_length': 2.645910780669145, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4991943', 'n_tokens_mistral': 1230, 'n_tokens_neox': 1183, 'n_words': 324}