<commit_before># encoding: utf-8
"""
Processing tools for main content.
"""
from urllib.parse import urlunsplit, urlparse
from collections import namedtuple

from bs4 import BeautifulSoup


# Compatible with tuples form urllib.parse
URL = namedtuple('URL', 'scheme netloc path query fragment')
URL.__new__.__defaults__ = ('http', '', '', '', '', '')


def clean_discourse_html(html, base_url):
    base_url_parts = urlparse(base_url)
    soup = BeautifulSoup(html, 'html.parser')

    for link_tag in soup.find_all('a'):
        link_tag['href'] = make_absolute_link(link_tag['href'], base_url_parts)

    for link_tag in soup.find_all('img'):
        link_tag['src'] = make_absolute_link(link_tag['src'], base_url_parts)

    return soup.prettify()


def make_absolute_link(discourse_link, base_url_parts):
    """Make any link from Discourse into an Absolute URL.

    Discourse provides link with a '//' prefix (e.g.
    //community.lsst.org/path) or relative, e.g. (/users/jsick).
    """
    url_parts = urlparse(discourse_link, scheme='http')

    # reconstruct URL
    if len(url_parts.netloc) == 0:
        # relative (local) url; add the scheme and netloc
        return urlunsplit(URL(scheme=url_parts.scheme,
                              netloc=base_url_parts.netloc,
                              path=url_parts.path))
    else:
        # treat as a global URL
        # this roundtrip process ensures the scheme is present
        return urlunsplit(URL(scheme=url_parts.scheme,
                              netloc=url_parts.netloc,
                              path=url_parts.path))<commit_msg>Catch exception in link cleaning<commit_after># encoding: utf-8
"""
Processing tools for main content.
"""
from urllib.parse import urlunsplit, urlparse
from collections import namedtuple

from bs4 import BeautifulSoup


# Compatible with tuples form urllib.parse
URL = namedtuple('URL', 'scheme netloc path query fragment')
URL.__new__.__defaults__ = ('http', '', '', '', '', '')


def clean_discourse_html(html, base_url):
    base_url_parts = urlparse(base_url)
    soup = BeautifulSoup(html, 'html.parser')

    for link_tag in soup.find_all('a'):
        try:
            link_tag['href'] = make_absolute_link(link_tag['href'],
                                                  base_url_parts)
        except KeyError:
            print('Missing href key in HTML cleanup; skipping')

    for link_tag in soup.find_all('img'):
        try:
            link_tag['src'] = make_absolute_link(link_tag['src'],
                                                 base_url_parts)
        except KeyError:
            print('Missing src key in HTML cleanup; skipping')

    return soup.prettify()


def make_absolute_link(discourse_link, base_url_parts):
    """Make any link from Discourse into an Absolute URL.

    Discourse provides link with a '//' prefix (e.g.
    //community.lsst.org/path) or relative, e.g. (/users/jsick).
    """
    url_parts = urlparse(discourse_link, scheme='http')

    # reconstruct URL
    if len(url_parts.netloc) == 0:
        # relative (local) url; add the scheme and netloc
        return urlunsplit(URL(scheme=url_parts.scheme,
                              netloc=base_url_parts.netloc,
                              path=url_parts.path))
    else:
        # treat as a global URL
        # this roundtrip process ensures the scheme is present
        return urlunsplit(URL(scheme=url_parts.scheme,
                              netloc=url_parts.netloc,
                              path=url_parts.path))
{'fraction_non_alphanumeric': 0.10369118061425754, 'fraction_numerical': 0.0016906170752324597, 'mean_word_length': 2.4940944881889764, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6119005', 'n_tokens_mistral': 1048, 'n_tokens_neox': 1015, 'n_words': 265}