<commit_before>from selector_finder import find_selector
from redis_client import RedisClient
from unidecode import unidecode
from bs4 import BeautifulSoup
from fetcher import fetch
from util import is_valid
import logging
import time
import re


redis_client = RedisClient()


def clean_soup(soup):
    """Removes some elements that may negatively affect the
    quality of headline extraction

    # Arguments
        soup: parsed html document
    """
    exclude_tags = ['style', 'script', '[document]', 'head', 'title']
    [s.extract() for s in soup(exclude_tags)]


def find_headline_element(soup, headline):
    """Finds the headline element on a page based on a headline hint.

    # Argument
        soup: parsed html page
        headline: headline hint to be used

    # Returns
        el: headline element (None if not found)
    """
    clean_soup(soup)
    # headline sometimes contains "..." at the end. We eliminate it.
    elems = soup(text=re.compile(re.escape(headline[:-4])))
    d = {}
    for el in elems:
        d[el.parent] = el.parent.text.strip()
    headline_elems = sorted(d, key=lambda k: len(d[k]))
    if len(headline_elems) > 0:
        return headline_elems[0]
    logging.debug('Headline "{}" not found'.format(unidecode(headline)))
    return None


def append_html(news):
    """Appends an html field to the news, only if the wayback_url is valid and
    the url does not already exist in the database.

    # Arguments
        news: news object as dictionary

    # Returns
        news: news object with or without html field
    """
    if is_valid(news, field='wayback_url'):
        if not redis_client.exists(news['url']):
            news['html'] = fetch(news['wayback_url'])
            return news
        logging.info('Skipping duplicate url: {}'.format(news['url']))
    return news


def append_headline_selector(news):
    """Appends the headline css selector field to the news, only if the html
    field exists and is valid.

    # Arguments
        news: news object as dictionary

    # Returns
        news: news object with or without headline css selector field
    """
    if is_valid(news, field='html'):
        soup = BeautifulSoup(news['html'], 'html.parser')
        headline_el = find_headline_element(soup, news['headline'])
        if headline_el:
            news['headline_selector'] = find_selector(soup, headline_el)
            return news
        logging.debug('Headline css selector could not be found!')
    return news


def get_base_url(lang='de'):
    """ Return the google news url for a specific language

    # Arguments
        lang: required language for google news
    # Returns
        url: corresponding google news url for the given language
    """
    if lang == 'de':
        return 'https://news.google.de/'
    else:
        raise ValueError('unsupported language {}'.format(lang))


def populate(lang='de'):
    keys = redis_client.keys()
    for key in keys:
        value = redis_client.get(key)
        html = fetch(value['wayback_url'])
        time.sleep(1)
        soup = BeautifulSoup(html, 'html.parser')
        headline_elems = soup.select(value['headline_selector'], None)
        if headline_elems:
            headline = headline_elems[0].text.strip()
        else:
            headline = None
        news = {}
        news['id'] = value['id']
        news['url'] = value['url']
        news['headline'] = headline
        news['wayback_url'] = value['wayback_url']
        yield news<commit_msg>added automatic body extraction<commit_after>from selector_finder import find_selector
from dragnet import content_extractor
from redis_client import RedisClient
from unidecode import unidecode
from bs4 import BeautifulSoup
from fetcher import fetch
from util import is_valid
import logging
import time
import re


redis_client = RedisClient()


def clean_soup(soup):
    """Removes some elements that may negatively affect the
    quality of headline extraction

    # Arguments
        soup: parsed html document
    """
    exclude_tags = ['style', 'script', '[document]', 'head', 'title']
    [s.extract() for s in soup(exclude_tags)]


def find_headline_element(soup, headline):
    """Finds the headline element on a page based on a headline hint.

    # Argument
        soup: parsed html page
        headline: headline hint to be used

    # Returns
        el: headline element (None if not found)
    """
    clean_soup(soup)
    # headline sometimes contains "..." at the end. We eliminate it.
    elems = soup(text=re.compile(re.escape(headline[:-4])))
    d = {}
    for el in elems:
        d[el.parent] = el.parent.text.strip()
    headline_elems = sorted(d, key=lambda k: len(d[k]))
    if len(headline_elems) > 0:
        return headline_elems[0]
    logging.debug('Headline "{}" not found'.format(unidecode(headline)))
    return None


def append_html(news):
    """Appends an html field to the news, only if the wayback_url is valid and
    the url does not already exist in the database.

    # Arguments
        news: news object as dictionary

    # Returns
        news: news object with or without html field
    """
    if is_valid(news, field='wayback_url'):
        if not redis_client.exists(news['url']):
            news['html'] = fetch(news['wayback_url'])
            return news
        logging.info('Skipping duplicate url: {}'.format(news['url']))
    return news


def append_headline_selector(news):
    """Appends the headline css selector field to the news, only if the html
    field exists and is valid.

    # Arguments
        news: news object as dictionary

    # Returns
        news: news object with or without headline css selector field
    """
    if is_valid(news, field='html'):
        soup = BeautifulSoup(news['html'], 'html.parser')
        headline_el = find_headline_element(soup, news['headline'])
        if headline_el:
            news['headline_selector'] = find_selector(soup, headline_el)
            return news
        logging.debug('Headline css selector could not be found!')
    return news


def get_base_url(lang='de'):
    """ Return the google news url for a specific language

    # Arguments
        lang: required language for google news
    # Returns
        url: corresponding google news url for the given language
    """
    if lang == 'de':
        return 'https://news.google.de/'
    else:
        raise ValueError('unsupported language {}'.format(lang))


def populate(lang='de'):
    keys = redis_client.keys()
    for key in keys:
        value = redis_client.get(key)
        html = fetch(value['wayback_url'])
        time.sleep(1)
        soup = BeautifulSoup(html, 'html.parser')
        headline_elems = soup.select(value['headline_selector'], None)
        if headline_elems:
            headline = headline_elems[0].text.strip()
        else:
            headline = None
        news = {}
        news['id'] = value['id']
        news['url'] = value['url']
        news['headline'] = headline
        news['wayback_url'] = value['wayback_url']
        news['body'] = content_extractor.analyze(html)
        news['html'] = html
        yield news
{'fraction_non_alphanumeric': 0.1039306196670863, 'fraction_numerical': 0.001678556441460344, 'mean_word_length': 2.805215540180947, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '306658', 'n_tokens_mistral': 2095, 'n_tokens_neox': 2031, 'n_words': 656}