<commit_before>#!/usr/bin/env python

import sys
import urllib
import locale
from datetime import datetime
from BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, UnicodeDammit, Tag
from utils import fetch_html_content, fetch_rss_content, count_words, make_soup_from_html_content


# for datetime conversions
if sys.platform in ['linux2', 'darwin', 'cygwin']:
    locale.setlocale(locale.LC_TIME, "fr_FR")


class ArticleData(object):
    def __init__(self, url, title, date, content, links, category, author, intro):
        self.url = url
        self.title = title
        self.date = date
        self.content = content
        self.links = links
        self.category = category
        self.author = author
        self.intro = intro



    def to_json(self):
        pass




def sanitize_fragment(fragment):
    if isinstance(fragment, Tag):
        return fragment.contents[0]
    else:
        return fragment

    
def sanitize_paragraph(paragraph):
    """removes image links, removes paragraphs, formatting"""
    return "".join([sanitize_fragment(fragment) for fragment in paragraph.contents])




def extract_content(story):
    header = story.find("div", {"id":"story_head"})
    story = story.find("div", {"id":"story_body"})

    paragraphs = story.findAll("p", recursive=False) 

    clean_paragraphs = [sanitize_paragraph(p) for p in paragraphs]
    
    return "".join(clean_paragraphs)
    

def extract_to_read_links_from_sidebar(sidebar):
    to_read_links_container = sidebar.find("div", {"id":"lire_aussi"})

    #sometimes, it does not exist at all
    if to_read_links_container:
        return [(link.get("href"), link.get("title"))
                for link in to_read_links_container.findAll("a")]
    else:
        return []


def extract_external_links_from_sidebar(sidebar):
    external_links_container = sidebar.find("div", {"id":"external"})

    #warning : weird javascript clickthru crap to process first
    if external_links_container:
        return [(link.get("onclick"), link.get("title"))
                for link in external_links_container.findAll("a")]
    else:
        return []



def extract_recent_links_from_soup(soup):
    #todo : check if those links are actually associated to the article
    recent_links_container = soup.find("div", {"id":"les_plus_recents"})
    if recent_links_container:
        return [(link.get("href"), link.contents[0])
                for link in recent_links_container.findAll("a", recursive=False) ]
    else:
        return []


def extract_links(soup):
    """
    Get the link lists for one news item, from the parsed html content.
    'Le Soir' has 3 kinds of links, but they're not all always there.
    """
    sidebar = soup.find("div", {'id':"st_top_center"})
    
    all_links = {"to_read":extract_to_read_links_from_sidebar(sidebar),
                 "external":extract_external_links_from_sidebar(sidebar),
                 "recent":extract_recent_links_from_soup(soup)}
    
    return all_links<commit_msg>[Le Soir] code layout<commit_after>#!/usr/bin/env python

import sys
import urllib
import locale
from datetime import datetime
from BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, UnicodeDammit, Tag
from utils import fetch_html_content, fetch_rss_content, count_words, make_soup_from_html_content


# for datetime conversions
if sys.platform in ['linux2', 'darwin', 'cygwin']:
    locale.setlocale(locale.LC_TIME, "fr_FR")


class ArticleData(object):
    def __init__(self, url, title, date, content, links, category, author, intro):
        self.url = url
        self.title = title
        self.date = date
        self.content = content
        self.links = links
        self.category = category
        self.author = author
        self.intro = intro


    def to_json(self):
        pass




def sanitize_fragment(fragment):
    if isinstance(fragment, Tag):
        return fragment.contents[0]
    else:
        return fragment

    
    
def sanitize_paragraph(paragraph):
    """removes image links, removes paragraphs, formatting"""
    return "".join([sanitize_fragment(fragment) for fragment in paragraph.contents])




def extract_content(story):
    header = story.find("div", {"id":"story_head"})
    story = story.find("div", {"id":"story_body"})

    paragraphs = story.findAll("p", recursive=False) 

    clean_paragraphs = [sanitize_paragraph(p) for p in paragraphs]
    
    return "".join(clean_paragraphs)
    


def extract_to_read_links_from_sidebar(sidebar):
    to_read_links_container = sidebar.find("div", {"id":"lire_aussi"})

    #sometimes, it does not exist at all
    if to_read_links_container:
        return [(link.get("href"), link.get("title"))
                for link in to_read_links_container.findAll("a")]
    else:
        return []

    

def extract_external_links_from_sidebar(sidebar):
    external_links_container = sidebar.find("div", {"id":"external"})

    #warning : weird javascript clickthru crap to process first
    if external_links_container:
        return [(link.get("onclick"), link.get("title"))
                for link in external_links_container.findAll("a")]
    else:
        return []




def extract_recent_links_from_soup(soup):
    #todo : check if those links are actually associated to the article
    recent_links_container = soup.find("div", {"id":"les_plus_recents"})
    if recent_links_container:
        return [(link.get("href"), link.contents[0])
                for link in recent_links_container.findAll("a", recursive=False) ]
    else:
        return []

    

def extract_links(soup):
    """
    Get the link lists for one news item, from the parsed html content.
    'Le Soir' has 3 kinds of links, but they're not all always there.
    """
    sidebar = soup.find("div", {'id':"st_top_center"})
    
    all_links = {"to_read":extract_to_read_links_from_sidebar(sidebar),
                 "external":extract_external_links_from_sidebar(sidebar),
                 "recent":extract_recent_links_from_soup(soup)}
    
    return all_links
{'fraction_non_alphanumeric': 0.10853739245532759, 'fraction_numerical': 0.0013236267372600927, 'mean_word_length': 3.374095513748191, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1165436', 'n_tokens_mistral': 1852, 'n_tokens_neox': 1743, 'n_words': 448}