<commit_before>import sys
from datetime import datetime, time
import locale
from itertools import chain
import codecs
from common.utils import fetch_content_from_url, make_soup_from_html_content, remove_text_formatting_markup_from_fragments, remove_text_formatting_markup
from common.utils import extract_plaintext_urls_from_text
from csxj.common.tagging import tag_URL, classify_and_tag, make_tagged_url, TaggedURL
from csxj.db.article import ArticleData


# for datetime conversions
if sys.platform in ['linux2', 'cygwin']:
    locale.setlocale(locale.LC_TIME, 'fr_FR.UTF8')
elif sys.platform in [ 'darwin']:
    locale.setlocale(locale.LC_TIME, 'fr_BE')


RTLINFO_OWN_NETLOC = 'www.rtl.be'

RTLINFO_INTERNAL_SITES = {
    'blogs.rtlinfo.be':['internal', 'blog'],
}

SOURCE_TITLE = "RTL Info"
SOURCE_NAME = "rtlinfo"

def extract_title(main_article):
    left_column = main_article.find('div', {'id':'leftCol'})
    title = left_column.find('h1', {'class':'rtl_font_weight_normal'})

    return remove_text_formatting_markup_from_fragments(title.contents)



def extract_date_and_time(main_article):
    date_container = main_article.find('span', {'class':'date'})
    date_string = date_container.contents[0].strip()

    date_bytestring = codecs.encode(date_string, 'utf-8')
    pub_date = datetime.strptime(date_bytestring, '%d %B %Y')

    time_container = date_container.find('span', {'class':'time'})
    time_string=  time_container.contents[0]
    h, m = [int(i) for i in time_string.split('h')]
    pub_time = time(h, m)

    return pub_date, pub_time



def extract_category(main_article):
    breadcrumb_container = main_article.find('div', {'id':'breadcrumb'})
    breadcrumbs = [''.join(link.contents) for link in  breadcrumb_container.findAll('a')]

    return breadcrumbs



def extract_external_links(main_article):
    container = main_article.find('div', {'class':'art_ext_links'})

    if container:
        link_list = container.ul
        items = link_list.findAll('li')
        urls_and_titles = [(i.a.get('href'), remove_text_formatting_markup_from_fragments(i.a.contents))  for i in items]

        tagged_urls = list()
        for url, title in urls_and_titles:
            tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
            tagged_urls.append(make_tagged_url(url, title, tags))
            
        return tagged_urls

    else:
        return []



def extract_related_links(main_article):
    container = main_article.find('div', {'class':'relatedArticles'})

    if container:
        left_list, right_list = container.findAll('ul')
        all_list_items = [link_list.findAll('li', recursive=False) for link_list in (left_list, right_list)]

        tagged_urls = list()
        for item in chain(*all_list_items):
            url, title = item.a.get('href'), remove_text_formatting_markup_from_fragments(item.a.contents)
            tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
            tags.add('associated')

            tagged_urls.append(make_tagged_url(url, title, tags))

        return tagged_urls
    else:
        return []


def extract_links(main_article):
    external_links = extract_external_links(main_article)
    related_links = extract_related_links(main_article)

    return external_links + related_links



def extract_usable_links(container):
    """
    Extracts all <a> elements, then filters out anything with no title and target.

    Why, will you ask me? I'm going to tell you why. Because *sometimes* that
    stupid cms allows for broken empty links to appear in the source.
    It's not even clickable. It's just there, doing nothing.

    With no target.
    With no title.
    With no purpose.
    With no soul.

    This is the saddest thing ever.
    """
    def is_usable(link):
        return link.get('href') and link.contents

    all_links = container.findAll('a')
    return [l for l in  all_links if is_usable(l)]



def extract_embedded_links_from_articlebody(article_body):
    embedded_links = list()

    for link in extract_usable_links(article_body):
        url = link.get('href')
        title = remove_text_formatting_markup_from_fragments(link.contents)
        tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
        tags.add('in text')
        embedded_links.append(make_tagged_url(url, title, tags))

    for embedded_video_frame in article_body.findAll('iframe'):
        url = embedded_video_frame.get('src')
        title = '[Video] {0}'.format(url)
        tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
        tags.union(['in text', 'embedded'])
        embedded_links.append(make_tagged_url(url, title, tags))
        
    return embedded_links



def extract_links_and_text_content(main_article):
    article_body = main_article.find('div', {'class':'articleBody rtl_margin_top_25'})

    embedded_links = extract_embedded_links_from_articlebody(article_body)

    all_paragraphs = article_body.findAll('p', recursive=False)
    cleaned_up_paragraphs = list()
    all_plaintext_urls = list()

    for p in all_paragraphs:
        paragraph = remove_text_formatting_markup_from_fragments(p.contents)
        plaintext_urls = extract_plaintext_urls_from_text(paragraph)
        for url in plaintext_urls:
            tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
            tags.add('in text', 'plaintext')
            all_plaintext_urls.append(make_tagged_url(url, url, tags))

        cleaned_up_paragraphs.append(paragraph)

    all_links = embedded_links+all_plaintext_urls
    return all_links, cleaned_up_paragraphs


def extract_intro(main_article):
    left_column = main_article.find('div', {'id':'leftCol'})
    intro_container = left_column.find('h2', recursive=False)

    if intro_container:
        intro = remove_text_formatting_markup_from_fragments(intro_container.contents)
    else:
        intro = None

    return intro<commit_msg>fix: set.add() is only for one element<commit_after>import sys
from datetime import datetime, time
import locale
from itertools import chain
import codecs
from common.utils import fetch_content_from_url, make_soup_from_html_content, remove_text_formatting_markup_from_fragments, remove_text_formatting_markup
from common.utils import extract_plaintext_urls_from_text
from csxj.common.tagging import tag_URL, classify_and_tag, make_tagged_url, TaggedURL
from csxj.db.article import ArticleData


# for datetime conversions
if sys.platform in ['linux2', 'cygwin']:
    locale.setlocale(locale.LC_TIME, 'fr_FR.UTF8')
elif sys.platform in [ 'darwin']:
    locale.setlocale(locale.LC_TIME, 'fr_BE')


RTLINFO_OWN_NETLOC = 'www.rtl.be'

RTLINFO_INTERNAL_SITES = {
    'blogs.rtlinfo.be':['internal', 'blog'],
}

SOURCE_TITLE = "RTL Info"
SOURCE_NAME = "rtlinfo"

def extract_title(main_article):
    left_column = main_article.find('div', {'id':'leftCol'})
    title = left_column.find('h1', {'class':'rtl_font_weight_normal'})

    return remove_text_formatting_markup_from_fragments(title.contents)



def extract_date_and_time(main_article):
    date_container = main_article.find('span', {'class':'date'})
    date_string = date_container.contents[0].strip()

    date_bytestring = codecs.encode(date_string, 'utf-8')
    pub_date = datetime.strptime(date_bytestring, '%d %B %Y')

    time_container = date_container.find('span', {'class':'time'})
    time_string=  time_container.contents[0]
    h, m = [int(i) for i in time_string.split('h')]
    pub_time = time(h, m)

    return pub_date, pub_time



def extract_category(main_article):
    breadcrumb_container = main_article.find('div', {'id':'breadcrumb'})
    breadcrumbs = [''.join(link.contents) for link in  breadcrumb_container.findAll('a')]

    return breadcrumbs



def extract_external_links(main_article):
    container = main_article.find('div', {'class':'art_ext_links'})

    if container:
        link_list = container.ul
        items = link_list.findAll('li')
        urls_and_titles = [(i.a.get('href'), remove_text_formatting_markup_from_fragments(i.a.contents))  for i in items]

        tagged_urls = list()
        for url, title in urls_and_titles:
            tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
            tagged_urls.append(make_tagged_url(url, title, tags))
            
        return tagged_urls

    else:
        return []



def extract_related_links(main_article):
    container = main_article.find('div', {'class':'relatedArticles'})

    if container:
        left_list, right_list = container.findAll('ul')
        all_list_items = [link_list.findAll('li', recursive=False) for link_list in (left_list, right_list)]

        tagged_urls = list()
        for item in chain(*all_list_items):
            url, title = item.a.get('href'), remove_text_formatting_markup_from_fragments(item.a.contents)
            tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
            tags.add('associated')

            tagged_urls.append(make_tagged_url(url, title, tags))

        return tagged_urls
    else:
        return []


def extract_links(main_article):
    external_links = extract_external_links(main_article)
    related_links = extract_related_links(main_article)

    return external_links + related_links



def extract_usable_links(container):
    """
    Extracts all <a> elements, then filters out anything with no title and target.

    Why, will you ask me? I'm going to tell you why. Because *sometimes* that
    stupid cms allows for broken empty links to appear in the source.
    It's not even clickable. It's just there, doing nothing.

    With no target.
    With no title.
    With no purpose.
    With no soul.

    This is the saddest thing ever.
    """
    def is_usable(link):
        return link.get('href') and link.contents

    all_links = container.findAll('a')
    return [l for l in  all_links if is_usable(l)]



def extract_embedded_links_from_articlebody(article_body):
    embedded_links = list()

    for link in extract_usable_links(article_body):
        url = link.get('href')
        title = remove_text_formatting_markup_from_fragments(link.contents)
        tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
        tags.add('in text')
        embedded_links.append(make_tagged_url(url, title, tags))

    for embedded_video_frame in article_body.findAll('iframe'):
        url = embedded_video_frame.get('src')
        title = '[Video] {0}'.format(url)
        tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
        tags.union(['in text', 'embedded'])
        embedded_links.append(make_tagged_url(url, title, tags))
        
    return embedded_links



def extract_links_and_text_content(main_article):
    article_body = main_article.find('div', {'class':'articleBody rtl_margin_top_25'})

    embedded_links = extract_embedded_links_from_articlebody(article_body)

    all_paragraphs = article_body.findAll('p', recursive=False)
    cleaned_up_paragraphs = list()
    all_plaintext_urls = list()

    for p in all_paragraphs:
        paragraph = remove_text_formatting_markup_from_fragments(p.contents)
        plaintext_urls = extract_plaintext_urls_from_text(paragraph)
        for url in plaintext_urls:
            tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
            tags.union(['in text', 'plaintext'])
            all_plaintext_urls.append(make_tagged_url(url, url, tags))

        cleaned_up_paragraphs.append(paragraph)

    all_links = embedded_links+all_plaintext_urls
    return all_links, cleaned_up_paragraphs


def extract_intro(main_article):
    left_column = main_article.find('div', {'id':'leftCol'})
    intro_container = left_column.find('h2', recursive=False)

    if intro_container:
        intro = remove_text_formatting_markup_from_fragments(intro_container.contents)
    else:
        intro = None

    return intro
{'fraction_non_alphanumeric': 0.09959197268715131, 'fraction_numerical': 0.0016654176034640686, 'mean_word_length': 4.01670843776107, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2615503', 'n_tokens_mistral': 4009, 'n_tokens_neox': 3838, 'n_words': 785}