<commit_before>from bs4 import BeautifulSoup
from boilerpipe.extract import Extractor


class ContentExtractor:

    EXTRACTOR_DEFAULT = 'DefaultExtractor'
    EXTRACTOR_ARTICLE = 'ArticleExtractor'
    EXTRACTOR_ARTICLE_SENTENCES = 'ArticleSentencesExtractor'
    EXTRACTOR_KEEP_EVERTYHING = 'KeepEverythingExtractor'
    EXTRACTOR_KEEP_EVERTYHING_WITH_MIN_KWORDS = 'KeepEverythingWithMinKWordsExtractor'
    EXTRACTOR_LARGEST = 'LargestContentExtractor'
    EXTRACTOR_NUM_WORDS_RULES = 'NumWordsRulesExtractor'
    EXTRACTOR_CANOLA = 'CanolaExtractor'

    @staticmethod
    def extract(html, extractor_type):
        """Extract text content using several method from html

        :param html:
        :param extractor_type:
        :return:
        """
        return Extractor(extractor=extractor_type, html=html)

def extract_encoding(bs4):
    encoding = bs4.original_encoding

    if not encoding:
        # Find encoding from meta charset
        charset = bs4.select('meta[charset]')

        if charset.__len__() > 0:
            encoding = charset[0]['charset']

    return encoding


def extract_favicon(bs4):
    """Extracting favicon url from BeautifulSoup object

    :param bs4: `BeautifulSoup`
    :return: `list` List of favicon urls
    """

    favicon = []

    selectors = [
        'link[rel="icon"]',
        'link[rel="Icon"]',
        'link[rel="ICON"]',
        'link[rel^="shortcut"]',
        'link[rel^="Shortcut"]',
        'link[rel^="SHORTCUT"]'
    ]

    for selector in selectors:

        icons = bs4.select(selector)

        if icons:

            for icon in icons:

                if icon.has_attr('href'):

                    favicon.append(icon['href'])

    return favicon


def extract_metas(bs4):
    """Extracting meta tags from BeautifulSoup object

    :param bs4: `BeautifulSoup`
    :return: `list` List of meta tags
    """

    meta_tags = []

    metas = bs4.select('meta')

    for meta in metas:

        meta_content = {}

        meta_attrs = [
            'charset',
            'name',
            'content',
            'property',
            'http-equiv',
            'itemprop'
        ]

        for attr in meta_attrs:
            if meta.has_attr(attr):
                meta_content.update({attr: meta[attr]})

        meta_tags.append(meta_content)

    return meta_tags


def extract_links(bs4):
    """Extracting links from BeautifulSoup object

    :param bs4: `BeautifulSoup`
    :return: `list` List of links
    """

    unique_links = list(set([anchor['href'] for anchor in bs4.select('a[href]') if anchor.has_attr('href')]))

    return [link for link in unique_links if link != '#']


def extract_original_links(base_url, bs4):
    """Extracting links that contains specific url from BeautifulSoup object

    :param base_url: `str` specific url that matched with the links
    :param bs4: `BeautifulSoup`
    :return: `list` List of links
    """

    return [anchor['href'] for anchor in bs4.select('a[href]')
            if anchor.has_attr('href')
            if anchor['href'].startswith(base_url)]


def extract_css_links(bs4):
    """Extracting css links from BeautifulSoup object

    :param bs4: `BeautifulSoup`
    :return: `list` List of links
    """

    real_css = [anchor['href'] for anchor in bs4.select('a[href]')
                if anchor.has_attr('href')
                if anchor['href'].endswith('.css')]

    css_link_tags = [anchor['href'] for anchor in bs4.select('link[type="text/css"]')
                     if anchor.has_attr('href')]

    return list(set(real_css+css_link_tags))


def extract_js_links(bs4):
    """Extracting js links from BeautifulSoup object

    :param bs4: `BeautifulSoup`
    :return: `list` List of links
    """

    real_js = [anchor['href'] for anchor in bs4.select('a[href]')
               if anchor.has_attr('href')
               if anchor['href'].endswith('.js')]

    js_tags = [anchor['src'] for anchor in bs4.select('script[type="text/javascript"]')
               if anchor.has_attr('src')]

    return list(set(real_js+js_tags))


def extract_resource_links(bs4):
    """Extracting resource links such as .js and .css from BeautifulSoup object

    :param bs4: `BeautifulSoup`
    :return: `list` List of resource links
    """

    return extract_js_links(bs4) + extract_css_links(bs4)


def extract_images(bs4, lazy_image_attribute=None):
    """If lazy attribute is supplied, find image url on that attribute

    :param bs4:
    :param lazy_image_attribute:<commit_msg>Fix extract links, js links and css links<commit_after>import re
from urllib.parse import urlparse
from bs4 import BeautifulSoup
from boilerpipe.extract import Extractor


class ContentExtractor:

    EXTRACTOR_DEFAULT = 'DefaultExtractor'
    EXTRACTOR_ARTICLE = 'ArticleExtractor'
    EXTRACTOR_ARTICLE_SENTENCES = 'ArticleSentencesExtractor'
    EXTRACTOR_KEEP_EVERTYHING = 'KeepEverythingExtractor'
    EXTRACTOR_KEEP_EVERTYHING_WITH_MIN_KWORDS = 'KeepEverythingWithMinKWordsExtractor'
    EXTRACTOR_LARGEST = 'LargestContentExtractor'
    EXTRACTOR_NUM_WORDS_RULES = 'NumWordsRulesExtractor'
    EXTRACTOR_CANOLA = 'CanolaExtractor'

    @staticmethod
    def extract(html, extractor_type):
        """Extract text content using several method from html

        :param html:
        :param extractor_type:
        :return:
        """
        return Extractor(extractor=extractor_type, html=html)

def extract_encoding(bs4):
    encoding = bs4.original_encoding

    if not encoding:
        # Find encoding from meta charset
        charset = bs4.select('meta[charset]')

        if charset.__len__() > 0:
            encoding = charset[0]['charset']

    return encoding


def extract_favicon(bs4):
    """Extracting favicon url from BeautifulSoup object

    :param bs4: `BeautifulSoup`
    :return: `list` List of favicon urls
    """

    favicon = []

    selectors = [
        'link[rel="icon"]',
        'link[rel="Icon"]',
        'link[rel="ICON"]',
        'link[rel^="shortcut"]',
        'link[rel^="Shortcut"]',
        'link[rel^="SHORTCUT"]'
    ]

    for selector in selectors:

        icons = bs4.select(selector)

        if icons:

            for icon in icons:

                if icon.has_attr('href'):

                    favicon.append(icon['href'])

    return favicon


def extract_metas(bs4):
    """Extracting meta tags from BeautifulSoup object

    :param bs4: `BeautifulSoup`
    :return: `list` List of meta tags
    """

    meta_tags = []

    metas = bs4.select('meta')

    for meta in metas:

        meta_content = {}

        meta_attrs = [
            'charset',
            'name',
            'content',
            'property',
            'http-equiv',
            'itemprop'
        ]

        for attr in meta_attrs:
            if meta.has_attr(attr):
                meta_content.update({attr: meta[attr]})

        meta_tags.append(meta_content)

    return meta_tags


def convert_invalid_url(url):
    """Convert invalid url with adding extra 'http://' schema into it

    :param url:
    :return:
    """
    regex_valid_url = re.compile(
        r'^(?:http|ftp)s?://' # http:// or https://
        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
        r'localhost|' #localhost...
        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
        r'(?::\d+)?' # optional port
        r'(?:/?|[/?]\S+)$', re.IGNORECASE)

    return url if regex_valid_url.match(url) else 'http://{}'.format(url)


def extract_links(bs4):
    """Extracting links from BeautifulSoup object

    :param bs4: `BeautifulSoup`
    :return: `list` List of links
    """

    unique_links = list(set([anchor['href'] for anchor in bs4.select('a[href]') if anchor.has_attr('href')]))

    # remove irrelevant link
    unique_links = [link for link in unique_links if link != '#']

    # convert invalid link with adding 'http' schema
    return [convert_invalid_url(link) for link in unique_links]


def extract_original_links(base_url, bs4):
    """Extracting links that contains specific url from BeautifulSoup object

    :param base_url: `str` specific url that matched with the links
    :param bs4: `BeautifulSoup`
    :return: `list` List of links
    """
    valid_url = convert_invalid_url(base_url)

    url = urlparse(valid_url)

    base_url = '{}://{}'.format(url.scheme, url.netloc)

    links = extract_links(bs4)

    return [anchor for anchor in links if anchor.startswith(base_url)]


def extract_css_links(bs4):
    """Extracting css links from BeautifulSoup object

    :param bs4: `BeautifulSoup`
    :return: `list` List of links
    """

    links = extract_links(bs4)

    real_css = [anchor for anchor in links if anchor.endswith('.css')]

    css_link_tags = [anchor['href'] for anchor in bs4.select('link[type="text/css"]')
                     if anchor.has_attr('href')]

    return list(set(real_css+css_link_tags))


def extract_js_links(bs4):
    """Extracting js links from BeautifulSoup object

    :param bs4: `BeautifulSoup`
    :return: `list` List of links
    """

    links = extract_links(bs4)

    real_js = [anchor for anchor in links if anchor.endswith('.js')]

    js_tags = [anchor['src'] for anchor in bs4.select('script[type="text/javascript"]')
               if anchor.has_attr('src')]

    return list(set(real_js+js_tags))


def extract_resource_links(bs4):
    """Extracting resource links such as .js and .css from BeautifulSoup object

    :param bs4: `BeautifulSoup`
    :return: `list` List of resource links
    """

    return extract_js_links(bs4) + extract_css_links(bs4)


def extract_images(bs4, lazy_image_attribute=None):
    """If lazy attribute is supplied, find image url on that attribute

    :param bs4:
    :param lazy_image_attribute:
{'fraction_non_alphanumeric': 0.12215764250025492, 'fraction_numerical': 0.008769246456612624, 'mean_word_length': 2.8462745098039215, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4466226', 'n_tokens_mistral': 3303, 'n_tokens_neox': 3080, 'n_words': 750}