<commit_before>from boilerpipe.extract import Extractor
from joblib import Memory
import logging as log
import numpy as np
import requests


def _get_entry_content(entry):
    contents = [
        entry.get("description", ""),
        entry.get("content", [{}])[0].get("value", ""),
        entry.get("summary", "")
    ]
    return contents[np.argmax(map(len, contents))]


class FailedExtraction(Exception):
    pass


# see
# http://tomazkovacic.com/blog/2011/06/09/evaluating-text-extraction-algorithms/
def _scrape(url=None, html=None):
    assert (url != html)
    if html is not None:
        try:
            return Extractor(extractor='DefaultExtractor', html=html)
        except Exception as e:
            log.warn("can't extract {html}".format(html=html))
    else:
        try:
            return Extractor(extractor='DefaultExtractor', url=url)
        except Exception as e:
            log.warn("Can't extract {url} with boilerpipe".format(url=url))
            return _scrape(html=requests.get(url).content)


_memory = Memory(cachedir="content-cache", verbose=1, bytes_limit=10**9)


@_memory.cache(ignore=["entry"])
def url2html(url, entry=None):
    try:
        html = _scrape(url=url).getHTML()
    except Exception as e:
        html = '<h1>{title}</h1>\n'.format(
            title=entry.title) + _get_entry_content(entry)
    assert (html)
    return html


@_memory.cache(ignore=["entry"])
def url2text(url, entry=None):
    try:  # scrape url
        text = _scrape(url=url).getText()
        if len(text) < 40:
            raise FailedExtraction
    except Exception:  # scrape entry
        try:  # scrape entry content
            entry_content = _scrape(html=_get_entry_content(entry)).getText()
        except Exception:
            entry_content = ''
        text = (entry.title or '') + "." + (entry_content or '')
    assert (text)
    return text


# def entry2text(entry):
#     return url2text(entry.link, entry)
#

# def entry2html(entry):
#     return url2html(entry.link, entry)<commit_msg>switch to logging<commit_after>from boilerpipe.extract import Extractor
from joblib import Memory
import logging as log
import numpy as np
import requests


def _get_entry_content(entry):
    contents = [
        entry.get("description", ""),
        entry.get("content", [{}])[0].get("value", ""),
        entry.get("summary", "")
    ]
    return contents[np.argmax(map(len, contents))]


class FailedExtraction(Exception):
    pass


# see
# http://tomazkovacic.com/blog/2011/06/09/evaluating-text-extraction-algorithms/
def _scrape(url=None, html=None):
    assert (url != html)
    if html is not None:
        try:
            return Extractor(extractor='DefaultExtractor', html=html)
        except Exception as e:
            log.warn("can't extract {html}".format(html=html))
    else:
        try:
            return Extractor(extractor='DefaultExtractor', url=url)
        except Exception as e:
            log.warn("Can't extract {url} with boilerpipe".format(url=url))
            return _scrape(html=requests.get(url).content)


_memory = Memory(cachedir="content-cache", verbose=1, bytes_limit=10**9)


@_memory.cache(ignore=["entry"])
def url2html(url, entry=None):
    try:
        html = _scrape(url=url).getHTML()
    except Exception as e:
        log.warn("{url} can't be scraped\n".format(url=url))
        html = '<h1>{title}</h1>\n'.format(
            title=entry.title) + _get_entry_content(entry)
    if not html:
        log.error("Could not extract any html for for {url}".format(url=url))
        raise FailedExtraction
    return html


@_memory.cache(ignore=["entry"])
def url2text(url, entry=None):
    try:  # scrape url
        text = _scrape(url=url).getText()
        if len(text) < 40:
            raise FailedExtraction
    except Exception:  # scrape entry
        try:  # scrape entry content
            entry_content = _scrape(html=_get_entry_content(entry)).getText()
        except Exception:
            entry_content = ''
        text = (entry.title or '') + "." + (entry_content or '')
    if not text:
        log.error("Could not extract any text for {url}".format(url=url))
        raise FailedExtraction
    return text


# def entry2text(entry):
#     return url2text(entry.link, entry)
#

# def entry2html(entry):
#     return url2html(entry.link, entry)
{'fraction_non_alphanumeric': 0.13291868233126009, 'fraction_numerical': 0.01059663671964985, 'mean_word_length': 2.929411764705882, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '729911', 'n_tokens_mistral': 1404, 'n_tokens_neox': 1325, 'n_words': 306}