<commit_before>import urlparse
import logging
import BeautifulSoup
import urllib
import zlib
from flexget.feed import Entry
from flexget.plugin import *
from flexget.utils.soup import get_soup
from flexget.plugins.cached_input import cached
from flexget.utils.tools import urlopener

log = logging.getLogger('html')


class InputHtml:
    """
        Parses urls from html page. Usefull on sites which have direct download
        links of any type (mp3, jpg, torrent, ...).

        Many anime-fansubbers do not provide RSS-feed, this works well in many cases.

        Configuration expects url parameter.

        Note: This returns ALL links on url so you need to configure filters
        to match only to desired content.
    """

    def validator(self):
        from flexget import validator
        root = validator.factory()
        root.accept('text')
        advanced = root.accept('dict')
        advanced.accept('url', key='url', required=True)
        advanced.accept('text', key='dump')
        advanced.accept('text', key='title_from')
        regexps = advanced.accept('list', key='links_re')
        regexps.accept('regexp')
        return root

    @cached('html', 'url')
    @internet(log)
    def on_feed_input(self, feed):
        config = feed.config['html']
        if not isinstance(config, dict):
            config = {}
        pageurl = feed.get_input_url('html')

        log.debug('InputPlugin html requesting url %s' % pageurl)

        page = urlopener(pageurl, log)
        soup = get_soup(page)
        log.debug('Detected encoding %s' % soup.originalEncoding)

        # dump received content into a file
        if 'dump' in config:
            name = config['dump']
            log.info('Dumping %s into %s' % (pageurl, name))
            data = soup.prettify()
            f = open(name, 'w')
            f.write(data)
            f.close()

        self.create_entries(feed, pageurl, soup, config)

    def create_entries(self, feed, pageurl, soup, config):

        queue = []
        duplicates = {}
        duplicate_limit = 4

        def title_exists(title):
            """Helper method. Return True if title is already added to entries"""
            for entry in queue:
                if entry['title'] == title:
                    return True

        for link in soup.findAll('a'):
            # not a valid link
            if not link.has_key('href'):
                continue
            # no content in the link
            if not link.contents:
                continue

            url = link['href']

            # get only links matching regexp
            regexps = config.get('links_re', None)
            if regexps:
                import re
                accept = False
                for regexp in regexps:
                    if re.search(regexp, url):
                        accept = True
                if not accept:
                    continue

            title = link.contents[0]

            # tag inside link
            if isinstance(title, BeautifulSoup.Tag):
                log.log(5, 'title is tag: %s' % title)
                continue

            # just unable to get any decent title
            if title is None:
                title = link.next.string
                if title is None:
                    continue

            # strip unicode white spaces
            title = title.replace(u'\u200B', u'').strip()

            if not title:
                continue

            # fix broken urls
            if url.startswith('//'):
                url = 'http:' + url
            elif not url.startswith('http://') or not url.startswith('https://'):
                url = urlparse.urljoin(pageurl, url)

            title_from = config.get('title_from', 'auto')
            if title_from == 'url':
                parts = urllib.splitquery(url[url.rfind('/')+1:])
                title = urllib.unquote_plus(parts[0])
                log.debug('title from url: %s' % title)
            elif title_from == 'title':
                if not link.has_key('title'):
                    log.warning('Link %s doesn\t have title attribute, ignored.')
                    continue
                title = link['title']
                log.debug('title from title: %s' % title)
            elif title_from == 'contents':
                pass<commit_msg>Removed `contents`, use `link` instead.<commit_after>import urlparse
import logging
import BeautifulSoup
import urllib
import zlib
from flexget.feed import Entry
from flexget.plugin import *
from flexget.utils.soup import get_soup
from flexget.plugins.cached_input import cached
from flexget.utils.tools import urlopener

log = logging.getLogger('html')


class InputHtml:
    """
        Parses urls from html page. Usefull on sites which have direct download
        links of any type (mp3, jpg, torrent, ...).

        Many anime-fansubbers do not provide RSS-feed, this works well in many cases.

        Configuration expects url parameter.

        Note: This returns ALL links on url so you need to configure filters
        to match only to desired content.
    """

    def validator(self):
        from flexget import validator
        root = validator.factory()
        root.accept('text')
        advanced = root.accept('dict')
        advanced.accept('url', key='url', required=True)
        advanced.accept('text', key='dump')
        advanced.accept('text', key='title_from')
        regexps = advanced.accept('list', key='links_re')
        regexps.accept('regexp')
        return root

    @cached('html', 'url')
    @internet(log)
    def on_feed_input(self, feed):
        config = feed.config['html']
        if not isinstance(config, dict):
            config = {}
        pageurl = feed.get_input_url('html')

        log.debug('InputPlugin html requesting url %s' % pageurl)

        page = urlopener(pageurl, log)
        soup = get_soup(page)
        log.debug('Detected encoding %s' % soup.originalEncoding)

        # dump received content into a file
        if 'dump' in config:
            name = config['dump']
            log.info('Dumping %s into %s' % (pageurl, name))
            data = soup.prettify()
            f = open(name, 'w')
            f.write(data)
            f.close()

        self.create_entries(feed, pageurl, soup, config)

    def create_entries(self, feed, pageurl, soup, config):

        queue = []
        duplicates = {}
        duplicate_limit = 4

        def title_exists(title):
            """Helper method. Return True if title is already added to entries"""
            for entry in queue:
                if entry['title'] == title:
                    return True

        for link in soup.findAll('a'):
            # not a valid link
            if not link.has_key('href'):
                continue
            # no content in the link
            if not link.contents:
                continue

            url = link['href']

            # get only links matching regexp
            regexps = config.get('links_re', None)
            if regexps:
                import re
                accept = False
                for regexp in regexps:
                    if re.search(regexp, url):
                        accept = True
                if not accept:
                    continue

            title = link.contents[0]

            # tag inside link
            if isinstance(title, BeautifulSoup.Tag):
                log.log(5, 'title is tag: %s' % title)
                continue

            # just unable to get any decent title
            if title is None:
                title = link.next.string
                if title is None:
                    continue

            # strip unicode white spaces
            title = title.replace(u'\u200B', u'').strip()

            if not title:
                continue

            # fix broken urls
            if url.startswith('//'):
                url = 'http:' + url
            elif not url.startswith('http://') or not url.startswith('https://'):
                url = urlparse.urljoin(pageurl, url)

            title_from = config.get('title_from', 'auto')
            if title_from == 'url':
                parts = urllib.splitquery(url[url.rfind('/')+1:])
                title = urllib.unquote_plus(parts[0])
                log.debug('title from url: %s' % title)
            elif title_from == 'title':
                if not link.has_key('title'):
                    log.warning('Link %s doesn\t have title attribute, ignored.')
                    continue
                title = link['title']
                log.debug('title from title: %s' % title)
{'fraction_non_alphanumeric': 0.0994443158138458, 'fraction_numerical': 0.002083815698078259, 'mean_word_length': 1.6508131328628413, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5911444', 'n_tokens_mistral': 2338, 'n_tokens_neox': 2222, 'n_words': 705}