<commit_before>import bleach
import bleach_whitelist
from bs4 import BeautifulSoup

def _canonical_link_predicate(tag):
    return tag.name == u'link' and \
        tag.has_attr('rel') and \
        u'canonical' in tag['rel']

def sanitize_html(raw_html, canonical_rel=None):
    """
    Arguments:

    unclean: raw html to be cleaned
    canonical_rel: optional fully qualified URL to set as canonical link.
    """
    # Strip tags to the few that we like
    clean = bleach.clean(raw_html,
        bleach_whitelist.markdown_tags,
        bleach_whitelist.markdown_attrs,
        strip=True)

    # Set anchor tags' targets
    clean = bleach.linkify(clean, callbacks=[
        bleach.callbacks.nofollow,
        bleach.callbacks.target_blank
    ])
    return clean

def set_canonical_rel(raw_html, canonical_rel):
    soup = BeautifulSoup(raw_html)
    canonical_tags = soup.find_all(_canonical_link_predicate)
    if canonical_tags:
        for tag in canonical_tags:
            tag['href'] = canonical_rel
    else:
        new_tag = soup.new_tag('link', rel='canonical', href=canonical_rel)
        head = soup.find('head')
        head.append(new_tag)
    return unicode(soup)
<commit_msg>Correct comments and method sigs for sanitizer<commit_after>import bleach
import bleach_whitelist
from bs4 import BeautifulSoup

def _canonical_link_predicate(tag):
    return tag.name == u'link' and \
        tag.has_attr('rel') and \
        u'canonical' in tag['rel']

def sanitize_html(raw_html):
    """
    Sanitize the given raw_html.
    """
    # Strip tags to the few that we like
    clean = bleach.clean(raw_html,
        bleach_whitelist.markdown_tags,
        bleach_whitelist.markdown_attrs,
        strip=True)

    # Set anchor tags' targets
    clean = bleach.linkify(clean, callbacks=[
        bleach.callbacks.nofollow,
        bleach.callbacks.target_blank
    ])
    return clean

def set_canonical_rel(raw_html, href):
    """
    Add or update <link rel='canonical'...> in the given html to the given
    href. Note that this has the side effect of appending html/head/body tags
    to the given html fragment if it doesn't already have them.
    """
    soup = BeautifulSoup(raw_html)
    canonical_tags = soup.find_all(_canonical_link_predicate)
    if canonical_tags:
        for tag in canonical_tags:
            tag['href'] = href
    else:
        new_tag = soup.new_tag('link', rel='canonical', href=href)
        head = soup.find('head')
        head.append(new_tag)
    return unicode(soup)

{'fraction_non_alphanumeric': 0.09033280507131537, 'fraction_numerical': 0.000792393026941363, 'mean_word_length': 2.9826498422712935, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4719685', 'n_tokens_mistral': 807, 'n_tokens_neox': 780, 'n_words': 203}